feat: 支持输入框内选择知识库和文件,优化选择交互体验

This commit is contained in:
wizardchen
2025-12-19 22:17:56 +08:00
committed by lyingbug
parent 485d8d7252
commit de1ffa7f7c
49 changed files with 2246 additions and 521 deletions
@@ -292,3 +292,58 @@ func (r *knowledgeRepository) CountKnowledgeByStatus(
return count, nil
}
// SearchKnowledge searches knowledge items by keyword across the tenant
// If keyword is empty, returns recent files
// Only returns documents from document-type knowledge bases (excludes FAQ)
// Returns (results, hasMore, error)
func (r *knowledgeRepository) SearchKnowledge(
ctx context.Context,
tenantID uint64,
keyword string,
offset, limit int,
) ([]*types.Knowledge, bool, error) {
// Use raw query to properly map knowledge_base_name
type KnowledgeWithKBName struct {
types.Knowledge
KnowledgeBaseName string `gorm:"column:knowledge_base_name"`
}
var results []KnowledgeWithKBName
query := r.db.WithContext(ctx).
Table("knowledges").
Select("knowledges.*, knowledge_bases.name as knowledge_base_name").
Joins("JOIN knowledge_bases ON knowledge_bases.id = knowledges.knowledge_base_id").
Where("knowledges.tenant_id = ?", tenantID).
Where("knowledge_bases.type = ?", types.KnowledgeBaseTypeDocument).
Where("knowledges.deleted_at IS NULL")
// If keyword is provided, filter by file_name or title
if keyword != "" {
query = query.Where("knowledges.file_name LIKE ? ", "%"+keyword+"%")
}
// Fetch limit+1 to check if there are more results
err := query.Order("knowledges.created_at DESC").
Offset(offset).
Limit(limit + 1).
Scan(&results).Error
if err != nil {
return nil, false, err
}
// Check if there are more results
hasMore := len(results) > limit
if hasMore {
results = results[:limit]
}
// Convert to []*types.Knowledge
knowledges := make([]*types.Knowledge, len(results))
for i, r := range results {
k := r.Knowledge
k.KnowledgeBaseName = r.KnowledgeBaseName
knowledges[i] = &k
}
return knowledges, hasMore, nil
}
@@ -38,6 +38,18 @@ func (r *knowledgeBaseRepository) GetKnowledgeBaseByID(ctx context.Context, id s
return &kb, nil
}
// GetKnowledgeBaseByIDs gets knowledge bases by multiple ids
func (r *knowledgeBaseRepository) GetKnowledgeBaseByIDs(ctx context.Context, ids []string) ([]*types.KnowledgeBase, error) {
if len(ids) == 0 {
return []*types.KnowledgeBase{}, nil
}
var kbs []*types.KnowledgeBase
if err := r.db.WithContext(ctx).Where("id IN ?", ids).Find(&kbs).Error; err != nil {
return nil, err
}
return kbs, nil
}
// ListKnowledgeBases lists all knowledge bases
func (r *knowledgeBaseRepository) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) {
var kbs []*types.KnowledgeBase
@@ -355,10 +355,16 @@ func (e *elasticsearchRepository) deleteByFieldList(ctx context.Context, field s
// getBaseConds Construct base Elasticsearch query conditions based on retrieval parameters
// It creates MUST conditions for required fields and MUST_NOT conditions for excluded fields
// KnowledgeBaseIDs and KnowledgeIDs use AND logic (search specific documents within knowledge bases)
// Returns a JSON string representing the query conditions
func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) string {
// Build MUST conditions (positive filters)
must := make([]map[string]interface{}, 0)
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
@@ -366,6 +372,13 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
},
})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, map[string]interface{}{
"terms": map[string]interface{}{
"knowledge_id.keyword": params.KnowledgeIDs,
},
})
}
// Build MUST_NOT conditions (negative filters)
mustNot := make([]map[string]interface{}, 0)
@@ -236,8 +236,14 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
// getBaseConds creates the base query conditions for retrieval operations
// Returns a slice of Query objects with must and must_not conditions
// KnowledgeBaseIDs and KnowledgeIDs use AND logic (search specific documents within knowledge bases)
func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) []types.Query {
must := []types.Query{}
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
@@ -245,6 +251,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
},
}})
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, types.Query{Terms: &types.TermsQuery{
TermsQuery: map[string]types.TermsQueryField{
"knowledge_id.keyword": params.KnowledgeIDs,
},
}})
}
mustNot := make([]types.Query, 0)
// Exclude disabled chunks (is_enabled = false)
// Note: Historical data without is_enabled field will be included (not matching must_not)
@@ -166,14 +166,25 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
conds := make([]clause.Expression, 0)
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
// Use standard SQL IN clause instead of @@@ operator for better performance with B-tree index
conds = append(conds, clause.IN{
Column: "knowledge_base_id",
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
})
}
if len(params.KnowledgeIDs) > 0 {
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge IDs: %v", params.KnowledgeIDs)
conds = append(conds, clause.IN{
Column: "knowledge_id",
Values: common.ToInterfaceSlice(params.KnowledgeIDs),
})
}
conds = append(conds, clause.Expr{
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
Vars: []interface{}{params.Query},
@@ -250,13 +261,15 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
whereParts = append(whereParts, fmt.Sprintf("dimension = $%d", len(allVars)+1))
allVars = append(allVars, dimension)
// Knowledge base filter
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge base IDs: %v",
params.KnowledgeBaseIDs,
)
// Build IN clause with proper placeholders
placeholders := make([]string, len(params.KnowledgeBaseIDs))
paramStart := len(allVars) + 1
for i := range params.KnowledgeBaseIDs {
@@ -266,6 +279,20 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
whereParts = append(whereParts, fmt.Sprintf("knowledge_base_id IN (%s)",
strings.Join(placeholders, ", ")))
}
if len(params.KnowledgeIDs) > 0 {
logger.GetLogger(ctx).Debugf(
"[Postgres] Filtering vector search by knowledge IDs: %v",
params.KnowledgeIDs,
)
placeholders := make([]string, len(params.KnowledgeIDs))
paramStart := len(allVars) + 1
for i := range params.KnowledgeIDs {
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
allVars = append(allVars, params.KnowledgeIDs[i])
}
whereParts = append(whereParts, fmt.Sprintf("knowledge_id IN (%s)",
strings.Join(placeholders, ", ")))
}
// is_enabled filter
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
@@ -433,9 +433,16 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi
// Only retrieve enabled chunks
must = append(must, qdrant.NewMatchBool(fieldIsEnabled, true))
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
// - If only KnowledgeBaseIDs: search entire knowledge bases
// - If only KnowledgeIDs: search specific documents
// - If both: search specific documents within the knowledge bases (AND)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
}
if len(params.KnowledgeIDs) > 0 {
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeID, params.KnowledgeIDs...))
}
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
@@ -445,10 +452,12 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...))
}
return &qdrant.Filter{
filter := &qdrant.Filter{
Must: must,
MustNot: mustNot,
}
return filter
}
// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type
+86 -3
View File
@@ -141,6 +141,13 @@ func (s *agentService) CreateAgentEngine(
}
}
// Get selected documents information (user @ mentioned documents)
selectedDocs, err := s.getSelectedDocumentInfos(ctx, config.KnowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to get selected document details: %v", err)
selectedDocs = []*agent.SelectedDocumentInfo{}
}
systemPromptTemplate := ""
if config.UseCustomSystemPrompt {
systemPromptTemplate = config.ResolveSystemPrompt(config.WebSearchEnabled)
@@ -153,6 +160,7 @@ func (s *agentService) CreateAgentEngine(
toolRegistry,
eventBus,
kbInfos,
selectedDocs,
contextManager,
sessionID,
systemPromptTemplate,
@@ -173,6 +181,29 @@ func (s *agentService) registerTools(
) error {
// If no specific tools allowed, register default tools
allowedTools := tools.DefaultAllowedTools()
// Filter out knowledge base tools if no knowledge bases or knowledge IDs are configured
hasKnowledge := len(config.KnowledgeBases) > 0 || len(config.KnowledgeIDs) > 0
if !hasKnowledge {
filteredTools := make([]string, 0)
kbTools := map[string]bool{
"knowledge_search": true,
"grep_chunks": true,
"list_knowledge_chunks": true,
"query_knowledge_graph": true,
"get_document_info": true,
"database_query": true,
}
for _, toolName := range allowedTools {
if !kbTools[toolName] {
filteredTools = append(filteredTools, toolName)
}
}
allowedTools = filteredTools
logger.Infof(ctx, "Pure Agent Mode: Knowledge base tools disabled due to empty configuration")
}
// If web search is enabled, add web_search to allowedTools
if config.WebSearchEnabled {
allowedTools = append(allowedTools, "web_search")
@@ -203,16 +234,17 @@ func (s *agentService) registerTools(
registry.RegisterTool(
tools.NewKnowledgeSearchTool(
s.knowledgeBaseService,
s.knowledgeService,
s.chunkService,
tenantID,
config.KnowledgeBases,
config.SearchTargets,
rerankModel,
chatModel,
s.cfg,
))
case "grep_chunks":
registry.RegisterTool(tools.NewGrepChunksTool(s.db, tenantID, config.KnowledgeBases))
logger.Infof(ctx, "Registered grep_chunks tool for tenant: %d", tenantID)
registry.RegisterTool(tools.NewGrepChunksTool(s.db, tenantID, config.KnowledgeBases, config.KnowledgeIDs))
logger.Infof(ctx, "Registered grep_chunks tool for tenant: %d, KBs: %d, KnowledgeIDs: %d", tenantID, len(config.KnowledgeBases), len(config.KnowledgeIDs))
case "list_knowledge_chunks":
registry.RegisterTool(tools.NewListKnowledgeChunksTool(tenantID, s.knowledgeService, s.chunkService))
case "query_knowledge_graph":
@@ -372,3 +404,54 @@ func (s *agentService) getKnowledgeBaseInfos(ctx context.Context, kbIDs []string
return kbInfos, nil
}
// getSelectedDocumentInfos retrieves detailed information for user-selected documents (via @ mention)
// This loads the actual content of the documents to include in the system prompt
func (s *agentService) getSelectedDocumentInfos(ctx context.Context, knowledgeIDs []string) ([]*agent.SelectedDocumentInfo, error) {
if len(knowledgeIDs) == 0 {
return []*agent.SelectedDocumentInfo{}, nil
}
// Get tenant ID from context
tenantID := uint64(0)
if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok {
tenantID = tid
}
// Fetch knowledge metadata
knowledges, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
if err != nil {
return nil, fmt.Errorf("failed to get knowledge batch: %w", err)
}
// Build map for quick lookup
knowledgeMap := make(map[string]*types.Knowledge)
for _, k := range knowledges {
if k != nil {
knowledgeMap[k.ID] = k
}
}
selectedDocs := make([]*agent.SelectedDocumentInfo, 0, len(knowledgeIDs))
for _, kid := range knowledgeIDs {
k, ok := knowledgeMap[kid]
if !ok {
logger.Warnf(ctx, "Selected knowledge %s not found", kid)
continue
}
docInfo := &agent.SelectedDocumentInfo{
KnowledgeID: k.ID,
KnowledgeBaseID: k.KnowledgeBaseID,
Title: k.Title,
FileName: k.FileName,
FileType: k.FileType,
}
selectedDocs = append(selectedDocs, docInfo)
}
logger.Infof(ctx, "Loaded %d selected documents metadata for prompt", len(selectedDocs))
return selectedDocs, nil
}
@@ -22,6 +22,7 @@ type PluginExtractEntity struct {
modelService interfaces.ModelService // Model service for calling large language models
template *types.PromptTemplateStructured // Template for generating prompts
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
knowledgeRepo interfaces.KnowledgeRepository
}
// NewPluginRewrite creates a new query rewriting plugin instance
@@ -30,12 +31,14 @@ func NewPluginExtractEntity(
eventManager *EventManager,
modelService interfaces.ModelService,
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
knowledgeRepo interfaces.KnowledgeRepository,
config *config.Config,
) *PluginExtractEntity {
res := &PluginExtractEntity{
modelService: modelService,
template: config.ExtractManager.ExtractEntity,
knowledgeBaseRepo: knowledgeBaseRepo,
knowledgeRepo: knowledgeRepo,
}
eventManager.Register(res)
return res
@@ -65,16 +68,68 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
return next()
}
kb, err := p.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chatManage.KnowledgeBaseID)
// Collect all knowledge base IDs to query
kbIDSet := make(map[string]struct{})
for _, id := range chatManage.KnowledgeBaseIDs {
kbIDSet[id] = struct{}{}
}
// If KnowledgeIDs is specified, retrieve them and collect their knowledge base IDs
// Also build a mapping from KnowledgeID to KnowledgeBaseID
knowledgeToKBMap := make(map[string]string)
if len(chatManage.KnowledgeIDs) > 0 {
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, chatManage.TenantID, chatManage.KnowledgeIDs)
if err != nil {
logger.Errorf(ctx, "failed to get knowledges: %v", err)
return next()
}
for _, k := range knowledges {
kbIDSet[k.KnowledgeBaseID] = struct{}{}
knowledgeToKBMap[k.ID] = k.KnowledgeBaseID
}
}
// Convert set to slice
allKBIDs := make([]string, 0, len(kbIDSet))
for id := range kbIDSet {
allKBIDs = append(allKBIDs, id)
}
// Batch retrieve all knowledge bases
kbs, err := p.knowledgeBaseRepo.GetKnowledgeBaseByIDs(ctx, allKBIDs)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
logger.Errorf(ctx, "failed to get knowledge bases: %v", err)
return next()
}
if kb.ExtractConfig == nil {
logger.Warnf(ctx, "failed to get extract config")
// Check if any knowledge base has ExtractConfig enabled and collect their IDs
enabledKBSet := make(map[string]struct{})
for _, kb := range kbs {
if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled {
enabledKBSet[kb.ID] = struct{}{}
}
}
if len(enabledKBSet) == 0 {
logger.Debugf(ctx, "no knowledge base has extract config enabled")
return next()
}
// Save enabled knowledge base IDs for later use in search_entity
enabledKBIDs := make([]string, 0, len(enabledKBSet))
for id := range enabledKBSet {
enabledKBIDs = append(enabledKBIDs, id)
}
chatManage.EntityKBIDs = enabledKBIDs
// Filter knowledgeToKBMap to only include files from enabled knowledge bases
entityKnowledge := make(map[string]string)
for knowledgeID, kbID := range knowledgeToKBMap {
if _, ok := enabledKBSet[kbID]; ok {
entityKnowledge[knowledgeID] = kbID
}
}
chatManage.EntityKnowledge = entityKnowledge
template := &types.PromptTemplateStructured{
Description: p.template.Description,
Examples: p.template.Examples,
@@ -66,35 +66,54 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
return ErrGetRerankModel.WithError(err)
}
// Prepare passages for reranking
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
"candidate_cnt": len(chatManage.SearchResult),
})
// Prepare passages for reranking (excluding DirectLoad results)
var passages []string
var candidatesToRerank []*types.SearchResult
var directLoadResults []*types.SearchResult
for _, result := range chatManage.SearchResult {
if result.MatchType == types.MatchTypeDirectLoad {
directLoadResults = append(directLoadResults, result)
pipelineInfo(ctx, "Rerank", "direct_load_skip", map[string]interface{}{
"chunk_id": result.ID,
})
continue
}
// 合并Content和ImageInfo的文本内容
passage := getEnrichedPassage(ctx, result)
passages = append(passages, passage)
candidatesToRerank = append(candidatesToRerank, result)
}
// Single rerank call with RewriteQuery, use threshold degradation if no results
originalThreshold := chatManage.RerankThreshold
rerankResp := p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages)
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
"total_cnt": len(chatManage.SearchResult),
"candidate_cnt": len(candidatesToRerank),
"direct_cnt": len(directLoadResults),
})
// If no results and threshold is high enough, try with lower threshold
if len(rerankResp) == 0 && originalThreshold > 0.3 {
degradedThreshold := originalThreshold * 0.7
if degradedThreshold < 0.3 {
degradedThreshold = 0.3
var rerankResp []rerank.RankResult
// Only call rerank model if there are candidates
if len(candidatesToRerank) > 0 {
// Single rerank call with RewriteQuery, use threshold degradation if no results
originalThreshold := chatManage.RerankThreshold
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)
// If no results and threshold is high enough, try with lower threshold
if len(rerankResp) == 0 && originalThreshold > 0.3 {
degradedThreshold := originalThreshold * 0.7
if degradedThreshold < 0.3 {
degradedThreshold = 0.3
}
pipelineInfo(ctx, "Rerank", "threshold_degrade", map[string]interface{}{
"original": originalThreshold,
"degraded": degradedThreshold,
})
chatManage.RerankThreshold = degradedThreshold
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)
// Restore original threshold
chatManage.RerankThreshold = originalThreshold
}
pipelineInfo(ctx, "Rerank", "threshold_degrade", map[string]interface{}{
"original": originalThreshold,
"degraded": degradedThreshold,
})
chatManage.RerankThreshold = degradedThreshold
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages)
// Restore original threshold
chatManage.RerankThreshold = originalThreshold
}
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
@@ -114,9 +133,14 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
for i := range chatManage.SearchResult {
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
}
reranked := make([]*types.SearchResult, 0, len(rerankResp))
reranked := make([]*types.SearchResult, 0, len(rerankResp)+len(directLoadResults))
// Process reranked results
for _, rr := range rerankResp {
sr := chatManage.SearchResult[rr.Index]
if rr.Index >= len(candidatesToRerank) {
continue
}
sr := candidatesToRerank[rr.Index]
base := sr.Score
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
modelScore := rr.RelevanceScore
@@ -130,6 +154,23 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
})
reranked = append(reranked, sr)
}
// Process direct load results (bypass rerank model, assume high relevance)
for _, sr := range directLoadResults {
base := sr.Score
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
// Assign high model score for direct load items
modelScore := 1.0
sr.Score = compositeScore(sr, modelScore, base)
pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{
"chunk_id": sr.ID,
"base_score": fmt.Sprintf("%.4f", base),
"model_score": fmt.Sprintf("%.4f", modelScore),
"final_score": fmt.Sprintf("%.4f", sr.Score),
"match_type": sr.MatchType,
})
reranked = append(reranked, sr)
}
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
chatManage.RerankResult = final
@@ -160,6 +201,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
// rerank performs the actual reranking operation with given query and passages
func (p *PluginRerank) rerank(ctx context.Context,
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
candidates []*types.SearchResult,
) []rerank.RankResult {
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
"query_variant": query,
@@ -179,21 +221,26 @@ func (p *PluginRerank) rerank(ctx context.Context,
"threshold": chatManage.RerankThreshold,
})
for i := range min(5, len(rerankResp)) {
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
"rank": i + 1,
"score": rerankResp[i].RelevanceScore,
"chunk_id": chatManage.SearchResult[rerankResp[i].Index].ID,
"match_type": chatManage.SearchResult[rerankResp[i].Index].MatchType,
"chunk_type": chatManage.SearchResult[rerankResp[i].Index].ChunkType,
"content": chatManage.SearchResult[rerankResp[i].Index].Content,
})
if rerankResp[i].Index < len(candidates) {
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
"rank": i + 1,
"score": rerankResp[i].RelevanceScore,
"chunk_id": candidates[rerankResp[i].Index].ID,
"match_type": candidates[rerankResp[i].Index].MatchType,
"chunk_type": candidates[rerankResp[i].Index].ChunkType,
"content": candidates[rerankResp[i].Index].Content,
})
}
}
// Filter results based on threshold with special handling for history matches
rankFilter := []rerank.RankResult{}
for _, result := range rerankResp {
if result.Index >= len(candidates) {
continue
}
th := chatManage.RerankThreshold
matchType := chatManage.SearchResult[result.Index].MatchType
matchType := candidates[result.Index].MatchType
if matchType == types.MatchTypeHistory {
th = math.Max(th-0.1, 0.5) // Lower threshold for history matches
}
@@ -19,6 +19,7 @@ import (
type PluginSearch struct {
knowledgeBaseService interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
chunkService interfaces.ChunkService
config *config.Config
webSearchService interfaces.WebSearchService
tenantService interfaces.TenantService
@@ -28,6 +29,7 @@ type PluginSearch struct {
func NewPluginSearch(eventManager *EventManager,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
config *config.Config,
webSearchService interfaces.WebSearchService,
tenantService interfaces.TenantService,
@@ -36,6 +38,7 @@ func NewPluginSearch(eventManager *EventManager,
res := &PluginSearch{
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
config: config,
webSearchService: webSearchService,
tenantService: tenantService,
@@ -54,35 +57,25 @@ func (p *PluginSearch) ActivationEvents() []types.EventType {
func (p *PluginSearch) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
// Get knowledge base IDs list
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
// Fall back to single knowledge base
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
pipelineInfo(ctx, "Search", "fallback_kb", map[string]interface{}{
"session_id": chatManage.SessionID,
"kb_id": chatManage.KnowledgeBaseID,
})
}
if len(knowledgeBaseIDs) == 0 {
// Check if we have search targets
if len(chatManage.SearchTargets) == 0 && len(chatManage.KnowledgeBaseIDs) == 0 && len(chatManage.KnowledgeIDs) == 0 {
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return ErrSearch.WithError(nil)
return nil
}
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"kb_ids": strings.Join(knowledgeBaseIDs, ","),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"search_targets": len(chatManage.SearchTargets),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
})
// Run KB search and web search concurrently
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
"kb_count": len(knowledgeBaseIDs),
"search_targets": len(chatManage.SearchTargets),
"embedding_top_k": chatManage.EmbeddingTopK,
"vector_threshold": chatManage.VectorThreshold,
"keyword_threshold": chatManage.KeywordThreshold,
@@ -92,10 +85,10 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
allResults := make([]*types.SearchResult, 0)
wg.Add(2)
// Goroutine 1: Knowledge base search (rewrite + processed)
// Goroutine 1: Knowledge base search using SearchTargets
go func() {
defer wg.Done()
kbResults := p.searchKnowledgeBases(ctx, knowledgeBaseIDs, chatManage)
kbResults := p.searchByTargets(ctx, chatManage)
if len(kbResults) > 0 {
mu.Lock()
allResults = append(allResults, kbResults...)
@@ -141,11 +134,11 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
})
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
expKwTh := chatManage.KeywordThreshold * 0.8
// Concurrent expansion retrieval across queries and KBs
// Concurrent expansion retrieval across queries and search targets
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
var muExp sync.Mutex
var wgExp sync.WaitGroup
jobs := len(expansions) * len(knowledgeBaseIDs)
jobs := len(expansions) * len(chatManage.SearchTargets)
capSem := 16
if jobs < capSem {
capSem = jobs
@@ -159,9 +152,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
"cap": capSem,
})
for _, q := range expansions {
for _, kbID := range knowledgeBaseIDs {
for _, target := range chatManage.SearchTargets {
wgExp.Add(1)
go func(q string, kbID string) {
go func(q string, t *types.SearchTarget) {
defer wgExp.Done()
sem <- struct{}{}
defer func() { <-sem }()
@@ -173,17 +166,21 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
DisableVectorMatch: true,
DisableKeywordsMatch: false,
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, kbID, paramsExp)
// Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge {
paramsExp.KnowledgeIDs = t.KnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp)
if err != nil {
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
"kb_id": kbID,
"kb_id": t.KnowledgeBaseID,
"error": err.Error(),
})
return
}
if len(res) > 0 {
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": kbID,
"kb_id": t.KnowledgeBaseID,
"query": q,
"hits": len(res),
})
@@ -191,7 +188,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
expResults = append(expResults, res...)
muExp.Unlock()
}
}(q, kbID)
}(q, target)
}
}
wgExp.Wait()
@@ -308,46 +305,89 @@ func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
// searchKnowledgeBases performs KB searches across KB IDs using RewriteQuery only
func (p *PluginSearch) searchKnowledgeBases(
// searchByTargets performs KB searches using pre-computed SearchTargets
// This is the main search method that uses the unified search targets
func (p *PluginSearch) searchByTargets(
ctx context.Context,
knowledgeBaseIDs []string,
chatManage *types.ChatManage,
) []*types.SearchResult {
// Build params for rewrite query
baseParams := types.SearchParams{
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
if len(chatManage.SearchTargets) == 0 {
return nil
}
var wg sync.WaitGroup
var mu sync.Mutex
var results []*types.SearchResult
// Search with rewrite query only (removed duplicate ProcessedQuery search)
for _, kbID := range knowledgeBaseIDs {
// Search each target concurrently
for _, target := range chatManage.SearchTargets {
wg.Add(1)
go func(knowledgeBaseID string) {
go func(t *types.SearchTarget) {
defer wg.Done()
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams)
// List of knowledge IDs to perform vector search on
// Default to all IDs in the target
searchKnowledgeIDs := t.KnowledgeIDs
// Try direct loading for specific knowledge targets
if t.Type == types.SearchTargetTypeKnowledge {
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
"skipped_ids": len(skippedIDs),
})
mu.Lock()
results = append(results, directResults...)
mu.Unlock()
}
// If all files were loaded directly, we don't need to search anything
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
return
}
// Otherwise, only search the files that were skipped (too large)
searchKnowledgeIDs = skippedIDs
}
// If no IDs left to search (and we are in Knowledge mode), we are done
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
return
}
// Build params for rewrite query
params := types.SearchParams{
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
}
// Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge {
params.KnowledgeIDs = searchKnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
if err != nil {
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": knowledgeBaseID,
"query": baseParams.QueryText,
"error": err.Error(),
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"query": params.QueryText,
"error": err.Error(),
})
return
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": knowledgeBaseID,
"hit_count": len(res),
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
}(kbID)
}(target)
}
wg.Wait()
@@ -358,6 +398,93 @@ func (p *PluginSearch) searchKnowledgeBases(
return results
}
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
if len(knowledgeIDs) == 0 {
return nil, nil
}
// Limit direct loading to avoid OOM or context overflow
// 50 chunks * ~500 chars/chunk ~= 25k chars
const maxTotalChunks = 50
var allChunks []*types.Chunk
var skippedIDs []string
loadedKnowledgeIDs := make(map[string]bool)
for _, kid := range knowledgeIDs {
// Optimization: Check chunk count first if possible?
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
skippedIDs = append(skippedIDs, kid)
continue
}
if len(allChunks)+len(chunks) > maxTotalChunks {
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
kid, len(allChunks), len(chunks), maxTotalChunks)
skippedIDs = append(skippedIDs, kid)
continue
}
allChunks = append(allChunks, chunks...)
loadedKnowledgeIDs[kid] = true
}
if len(allChunks) == 0 {
return nil, skippedIDs
}
// Fetch Knowledge metadata
var uniqueKIDs []string
for kid := range loadedKnowledgeIDs {
uniqueKIDs = append(uniqueKIDs, kid)
}
knowledgeMap := make(map[string]*types.Knowledge)
if len(uniqueKIDs) > 0 {
knowledges, err := p.knowledgeService.GetKnowledgeBatch(ctx, tenantID, uniqueKIDs)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
// Continue without metadata
} else {
for _, k := range knowledges {
knowledgeMap[k.ID] = k
}
}
}
var results []*types.SearchResult
for _, chunk := range allChunks {
res := &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
Score: 1.0, // Maximum score for direct matches
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
MatchType: types.MatchTypeDirectLoad,
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
ChunkMetadata: chunk.Metadata,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
}
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
res.KnowledgeTitle = k.Title
res.KnowledgeFilename = k.FileName
res.KnowledgeSource = k.Source
res.Metadata = k.GetMetadata()
}
results = append(results, res)
}
return results, skippedIDs
}
// searchWebIfEnabled executes web search when enabled and returns converted results
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil || chatManage.TenantID <= 0 {
@@ -47,50 +47,81 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
return next()
}
// Get knowledge base IDs list
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
logger.Infof(ctx, "No KnowledgeBaseIDs provided, falling back to single KB: %s", chatManage.KnowledgeBaseID)
}
// Use EntityKBIDs (knowledge bases with ExtractConfig enabled)
knowledgeBaseIDs := chatManage.EntityKBIDs
// Use EntityKnowledge (KnowledgeID -> KnowledgeBaseID mapping for graph-enabled files)
entityKnowledge := chatManage.EntityKnowledge
if len(knowledgeBaseIDs) == 0 {
logger.Warnf(ctx, "No knowledge base IDs available for entity search")
if len(knowledgeBaseIDs) == 0 && len(entityKnowledge) == 0 {
logger.Warnf(ctx, "No knowledge base IDs or knowledge IDs with ExtractConfig enabled for entity search")
return next()
}
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
// Parallel search across multiple knowledge bases
// Parallel search across multiple knowledge bases and individual files
var wg sync.WaitGroup
var mu sync.Mutex
var allNodes []*types.GraphNode
var allRelations []*types.GraphRelation
for _, kbID := range knowledgeBaseIDs {
wg.Add(1)
go func(knowledgeBaseID string) {
defer wg.Done()
// If specific KnowledgeIDs are provided, search by individual files
if len(entityKnowledge) > 0 {
logger.Infof(ctx, "Searching entities across %d knowledge file(s)", len(entityKnowledge))
for knowledgeID, kbID := range entityKnowledge {
wg.Add(1)
go func(knowledgeBaseID, knowledgeID string) {
defer wg.Done()
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
return
}
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{
KnowledgeBase: knowledgeBaseID,
Knowledge: knowledgeID,
}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search entity in Knowledge %s: %v", knowledgeID, err)
return
}
logger.Infof(
ctx,
"KB %s entity search result count: %d nodes, %d relations",
knowledgeBaseID,
len(graph.Node),
len(graph.Relation),
)
logger.Infof(
ctx,
"Knowledge %s entity search result count: %d nodes, %d relations",
knowledgeID,
len(graph.Node),
len(graph.Relation),
)
mu.Lock()
allNodes = append(allNodes, graph.Node...)
allRelations = append(allRelations, graph.Relation...)
mu.Unlock()
}(kbID)
mu.Lock()
allNodes = append(allNodes, graph.Node...)
allRelations = append(allRelations, graph.Relation...)
mu.Unlock()
}(kbID, knowledgeID)
}
} else {
// Otherwise, search by knowledge base
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
for _, kbID := range knowledgeBaseIDs {
wg.Add(1)
go func(knowledgeBaseID string) {
defer wg.Done()
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
return
}
logger.Infof(
ctx,
"KB %s entity search result count: %d nodes, %d relations",
knowledgeBaseID,
len(graph.Node),
len(graph.Relation),
)
mu.Lock()
allNodes = append(allNodes, graph.Node...)
allRelations = append(allRelations, graph.Relation...)
mu.Unlock()
}(kbID)
}
}
wg.Wait()
@@ -35,6 +35,7 @@ func NewPluginSearchParallel(
eventManager *EventManager,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
config *config.Config,
webSearchService interfaces.WebSearchService,
tenantService interfaces.TenantService,
@@ -47,6 +48,7 @@ func NewPluginSearchParallel(
searchPlugin := &PluginSearch{
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
config: config,
webSearchService: webSearchService,
tenantService: tenantService,
+6 -7
View File
@@ -266,7 +266,6 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
StartTime: time.Now(),
},
Params: &types.ChatManage{
KnowledgeBaseID: knowledgeBaseID,
VectorThreshold: e.config.Conversation.VectorThreshold,
KeywordThreshold: e.config.Conversation.KeywordThreshold,
EmbeddingTopK: e.config.Conversation.EmbeddingTopK,
@@ -311,7 +310,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
logger.Info(newCtx, "Evaluation task status set to running")
// Execute actual evaluation
if err := e.EvalDataset(newCtx, detail); err != nil {
if err := e.EvalDataset(newCtx, detail, knowledgeBaseID); err != nil {
detail.Task.Status = types.EvaluationStatueFailed
detail.Task.ErrMsg = err.Error()
logger.Errorf(newCtx, "Evaluation task failed: %v, task ID: %s", err, taskID)
@@ -329,7 +328,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
// EvalDataset performs the actual evaluation of a dataset
// Processes each QA pair in parallel and records metrics
func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.EvaluationDetail) error {
func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.EvaluationDetail, knowledgeBaseID string) error {
logger.Info(ctx, "Start evaluating dataset")
logger.Infof(ctx, "Task ID: %s, Dataset ID: %s", detail.Task.ID, detail.Task.DatasetID)
@@ -352,7 +351,7 @@ func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.Evalu
logger.Infof(ctx, "Creating knowledge from %d passages", len(passages))
// Create knowledge base from passages
knowledge, err := e.knowledgeService.CreateKnowledgeFromPassage(ctx, detail.Params.KnowledgeBaseID, passages)
knowledge, err := e.knowledgeService.CreateKnowledgeFromPassage(ctx, knowledgeBaseID, passages)
if err != nil {
logger.Errorf(ctx, "Failed to create knowledge from passages: %v", err)
return err
@@ -366,12 +365,12 @@ func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.Evalu
logger.Errorf(ctx, "Failed to delete knowledge: %v, knowledge ID: %s", err, knowledge.ID)
}
logger.Infof(ctx, "Cleaning up resources - deleting knowledge base: %s", detail.Params.KnowledgeBaseID)
if err := e.knowledgeBaseService.DeleteKnowledgeBase(ctx, detail.Params.KnowledgeBaseID); err != nil {
logger.Infof(ctx, "Cleaning up resources - deleting knowledge base: %s", knowledgeBaseID)
if err := e.knowledgeBaseService.DeleteKnowledgeBase(ctx, knowledgeBaseID); err != nil {
logger.Errorf(
ctx,
"Failed to delete knowledge base: %v, knowledge base ID: %s",
err, detail.Params.KnowledgeBaseID,
err, knowledgeBaseID,
)
}
}()
+10
View File
@@ -5766,3 +5766,13 @@ func (s *knowledgeService) getOrCreateTagInTarget(
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dstKnowledgeBaseID)
return newTag.ID
}
// SearchKnowledge searches knowledge items by keyword across the tenant
func (s *knowledgeService) SearchKnowledge(ctx context.Context, keyword string, offset, limit int) ([]*types.Knowledge, bool, error) {
tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64)
if !ok {
return nil, false, werrors.NewUnauthorizedError("Tenant ID not found in context")
}
return s.repo.SearchKnowledge(ctx, tenantID, keyword, offset, limit)
}
@@ -491,6 +491,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
TopK: matchCount,
Threshold: params.VectorThreshold,
RetrieverType: types.VectorRetrieverType,
KnowledgeIDs: params.KnowledgeIDs,
}
// For FAQ knowledge base, use FAQ index
@@ -512,6 +513,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
TopK: matchCount,
Threshold: params.KeywordThreshold,
RetrieverType: types.KeywordsRetrieverType,
KnowledgeIDs: params.KnowledgeIDs,
})
logger.Info(ctx, "Keyword retrieval parameters setup completed")
}
+158 -52
View File
@@ -43,6 +43,7 @@ type sessionService struct {
agentService interfaces.AgentService // Service for agent operations
sessionStorage llmcontext.ContextStorage // Session storage
knowledgeService interfaces.KnowledgeService // Service for knowledge operations
chunkService interfaces.ChunkService // Service for chunk operations
redisClient *redis.Client // Redis client for temp KB state
}
@@ -52,6 +53,7 @@ func NewSessionService(cfg *config.Config,
messageRepo interfaces.MessageRepository,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
modelService interfaces.ModelService,
tenantService interfaces.TenantService,
eventManager *chatpipline.EventManager,
@@ -65,6 +67,7 @@ func NewSessionService(cfg *config.Config,
messageRepo: messageRepo,
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
modelService: modelService,
tenantService: tenantService,
eventManager: eventManager,
@@ -394,6 +397,7 @@ func (s *sessionService) KnowledgeQA(
session *types.Session,
query string,
knowledgeBaseIDs []string,
knowledgeIDs []string,
assistantMessageID string,
summaryModelID string,
webSearchEnabled bool,
@@ -414,14 +418,13 @@ func (s *sessionService) KnowledgeQA(
logger.Infof(ctx, "No knowledge base IDs provided, using session default: %s", session.KnowledgeBaseID)
} else {
logger.Warnf(ctx, "Session has no associated knowledge base, session ID: %s", session.ID)
return errors.New("session has no knowledge base")
}
}
logger.Infof(ctx, "Using knowledge bases: %v", knowledgeBaseIDs)
// Determine chat model ID: prioritize request's summaryModelID, then Remote models
chatModelID, err := s.selectChatModelIDWithOverride(ctx, session, knowledgeBaseIDs, summaryModelID)
chatModelID, err := s.selectChatModelIDWithOverride(ctx, session, knowledgeBaseIDs, knowledgeIDs, summaryModelID)
if err != nil {
return err
}
@@ -511,20 +514,29 @@ func (s *sessionService) KnowledgeQA(
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
}
// Build unified search targets (computed once, used throughout pipeline)
searchTargets, err := s.buildSearchTargets(ctx, session.TenantID, knowledgeBaseIDs, knowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to build search targets: %v", err)
}
// Create chat management object with session settings
logger.Infof(
ctx,
"Creating chat manage object, knowledge base IDs: %v, chat model ID: %s",
"Creating chat manage object, knowledge base IDs: %v, knowledge IDs: %v, chat model ID: %s, search targets: %d",
knowledgeBaseIDs,
knowledgeIDs,
chatModelID,
len(searchTargets),
)
chatManage := &types.ChatManage{
Query: query,
RewriteQuery: query,
SessionID: session.ID,
MessageID: assistantMessageID, // NEW: For event emission in pipeline
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
MessageID: assistantMessageID, // NEW: For event emission in pipeline
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
KnowledgeIDs: knowledgeIDs, // Specific knowledge (file) IDs
SearchTargets: searchTargets, // Pre-computed search targets
VectorThreshold: vectorThreshold,
KeywordThreshold: keywordThreshold,
EmbeddingTopK: embeddingTopK,
@@ -546,9 +558,22 @@ func (s *sessionService) KnowledgeQA(
EnableQueryExpansion: enableQueryExpansion,
}
// Determine pipeline based on knowledge bases availability
// If no knowledge bases are selected, use pure chat pipeline
var pipeline []types.EventType
if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) == 0 {
logger.Info(ctx, "No knowledge bases selected, using chat_stream pipeline")
pipeline = types.Pipline["chat_stream"]
// For pure chat, UserContent is the Query (since INTO_CHAT_MESSAGE is skipped)
chatManage.UserContent = query
} else {
logger.Info(ctx, "Knowledge bases selected, using rag_stream pipeline")
pipeline = types.Pipline["rag_stream"]
}
// Start knowledge QA event processing
logger.Info(ctx, "Triggering knowledge base question answering event")
err = s.KnowledgeQAByEvent(ctx, chatManage, types.Pipline["rag_stream"])
logger.Info(ctx, "Triggering question answering event")
err = s.KnowledgeQAByEvent(ctx, chatManage, pipeline)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"session_id": session.ID,
@@ -592,6 +617,7 @@ func (s *sessionService) selectChatModelIDWithOverride(
ctx context.Context,
session *types.Session,
knowledgeBaseIDs []string,
knowledgeIDs []string,
summaryModelID string,
) (string, error) {
// First, check if request has summaryModelID override
@@ -612,33 +638,47 @@ func (s *sessionService) selectChatModelIDWithOverride(
}
// If no valid override, use default selection logic
return s.selectChatModelID(ctx, session, knowledgeBaseIDs)
return s.selectChatModelID(ctx, session, knowledgeBaseIDs, knowledgeIDs)
}
// selectChatModelID selects the appropriate chat model ID with priority for Remote models
// Priority order:
// 1. Session's SummaryModelID if it's a Remote model
// 2. First knowledge base with a Remote model
// 2. First knowledge base with a Remote model (from knowledgeBaseIDs or derived from knowledgeIDs)
// 3. Session's SummaryModelID (if not Remote)
// 4. First knowledge base's SummaryModelID
func (s *sessionService) selectChatModelID(
ctx context.Context,
session *types.Session,
knowledgeBaseIDs []string,
knowledgeIDs []string,
) (string, error) {
// First, check if session has a SummaryModelID and if it's a Remote model
if session.SummaryModelID != "" {
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
if err == nil && model != nil && model.Source == types.ModelSourceRemote {
logger.Infof(ctx, "Using session's Remote summary model: %s", session.SummaryModelID)
return session.SummaryModelID, nil
} else if err == nil && model != nil {
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+
"checking knowledge bases for Remote models", session.SummaryModelID)
return session.SummaryModelID, nil
}
// If no knowledge base IDs but have knowledge IDs, derive KB IDs from knowledge IDs
if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) > 0 {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
knowledgeList, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to get knowledge batch for model selection: %v", err)
} else {
// Collect unique KB IDs from knowledge items
kbIDSet := make(map[string]bool)
for _, k := range knowledgeList {
if k != nil && k.KnowledgeBaseID != "" {
kbIDSet[k.KnowledgeBaseID] = true
}
}
for kbID := range kbIDSet {
knowledgeBaseIDs = append(knowledgeBaseIDs, kbID)
}
logger.Infof(ctx, "Derived %d knowledge base IDs from %d knowledge IDs for model selection",
len(knowledgeBaseIDs), len(knowledgeIDs))
}
}
// If no Remote model found from session, check knowledge bases for Remote models
if len(knowledgeBaseIDs) > 0 {
// Try to find a knowledge base with Remote model
@@ -683,12 +723,79 @@ func (s *sessionService) selectChatModelID(
}
}
// No knowledge bases - use session's SummaryModelID if available
if session.SummaryModelID != "" {
logger.Infof(ctx, "No knowledge bases, using session's summary model: %s", session.SummaryModelID)
return session.SummaryModelID, nil
}
logger.Error(ctx, "No chat model ID available")
return "", errors.New(
"no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID",
"no chat model ID available: session has no SummaryModelID and no knowledge bases configured",
)
}
// buildSearchTargets computes the unified search targets from knowledgeBaseIDs and knowledgeIDs
// This is called once at the request entry point to avoid repeated queries later in the pipeline
// Logic:
// - For each knowledgeBaseID: create a SearchTargetTypeKnowledgeBase target
// - For each knowledgeID: find its knowledgeBaseID, if the KB is already in the list, skip (covered by full KB search)
// otherwise create a SearchTargetTypeKnowledge target grouped by KB
func (s *sessionService) buildSearchTargets(
ctx context.Context,
tenantID uint64,
knowledgeBaseIDs []string,
knowledgeIDs []string,
) (types.SearchTargets, error) {
var targets types.SearchTargets
// Track which KBs are fully searched
fullKBSet := make(map[string]bool)
for _, kbID := range knowledgeBaseIDs {
fullKBSet[kbID] = true
targets = append(targets, &types.SearchTarget{
Type: types.SearchTargetTypeKnowledgeBase,
KnowledgeBaseID: kbID,
})
}
// Process individual knowledge IDs
if len(knowledgeIDs) > 0 {
knowledgeList, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to get knowledge batch for search targets: %v", err)
return targets, nil // Return what we have, don't fail
}
// Group knowledge IDs by their KB, excluding those already covered by full KB search
kbToKnowledgeIDs := make(map[string][]string)
for _, k := range knowledgeList {
if k == nil || k.KnowledgeBaseID == "" {
continue
}
// Skip if this KB is already fully searched
if fullKBSet[k.KnowledgeBaseID] {
continue
}
kbToKnowledgeIDs[k.KnowledgeBaseID] = append(kbToKnowledgeIDs[k.KnowledgeBaseID], k.ID)
}
// Create SearchTargetTypeKnowledge targets for each KB with specific files
for kbID, kidList := range kbToKnowledgeIDs {
targets = append(targets, &types.SearchTarget{
Type: types.SearchTargetTypeKnowledge,
KnowledgeBaseID: kbID,
KnowledgeIDs: kidList,
})
}
}
logger.Infof(ctx, "Built %d search targets: %d full KB, %d partial KB",
len(targets), len(knowledgeBaseIDs), len(targets)-len(knowledgeBaseIDs))
return targets, nil
}
// KnowledgeQAByEvent processes knowledge QA through a series of events in the pipeline
func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
chatManage *types.ChatManage, eventList []types.EventType,
@@ -697,8 +804,8 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
defer span.End()
logger.Info(ctx, "Start processing knowledge base question answering through events")
logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, knowledge base ID: %s, query: %s",
chatManage.SessionID, chatManage.KnowledgeBaseID, chatManage.Query)
logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s",
chatManage.SessionID, chatManage.Query)
// Prepare method list for logging and tracing
methods := []string{}
@@ -769,7 +876,6 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
chatManage := &types.ChatManage{
Query: query,
RewriteQuery: query,
KnowledgeBaseID: knowledgeBaseID,
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
@@ -800,10 +906,10 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
// Use specific event list, only including retrieval-related events, not LLM summarization
searchEvents := []types.EventType{
types.CHUNK_SEARCH, // Vector search
types.CHUNK_RERANK, // Rerank search results
types.CHUNK_MERGE, // Merge search results
types.FILTER_TOP_K, // Filter top K results
types.CHUNK_SEARCH, // Vector search
types.CHUNK_RERANK, // Rerank search results
types.CHUNK_MERGE, // Merge search results
types.FILTER_TOP_K, // Filter top K results
}
ctx, span := tracing.ContextWithSpan(ctx, "SessionService.SearchKnowledge")
@@ -895,13 +1001,14 @@ func (s *sessionService) AgentQA(
// Create runtime AgentConfig by merging session and tenant configs
// Tenant config provides the runtime parameters (MaxIterations, Temperature, Tools, Models)
// Session config provides KnowledgeBases
// Session config provides KnowledgeBases and KnowledgeIDs
agentConfig := &types.AgentConfig{
MaxIterations: tenantInfo.AgentConfig.MaxIterations,
ReflectionEnabled: tenantInfo.AgentConfig.ReflectionEnabled,
AllowedTools: tools.DefaultAllowedTools(),
Temperature: tenantInfo.AgentConfig.Temperature,
KnowledgeBases: session.AgentConfig.KnowledgeBases, // Use session's knowledge bases
KnowledgeIDs: session.AgentConfig.KnowledgeIDs, // Use session's knowledge IDs (individual documents)
WebSearchEnabled: session.AgentConfig.WebSearchEnabled, // Web search enabled from session config
}
@@ -919,43 +1026,42 @@ func (s *sessionService) AgentQA(
logger.Infof(ctx, "Merged agent config from tenant %d and session %s", tenantInfo.ID, sessionID)
// Log knowledge IDs if present
if len(agentConfig.KnowledgeIDs) > 0 {
logger.Infof(ctx, "Agent configured with %d individual knowledge ID(s): %v",
len(agentConfig.KnowledgeIDs), agentConfig.KnowledgeIDs)
}
// Determine knowledge bases for agent
// Priority: Session.AgentConfig.KnowledgeBases > Session.KnowledgeBaseID > All tenant knowledge bases
if len(agentConfig.KnowledgeBases) == 0 {
// Exception: If KnowledgeIDs are specified, don't auto-add all KBs (let buildSearchTargets handle it)
if len(agentConfig.KnowledgeBases) == 0 && len(agentConfig.KnowledgeIDs) == 0 {
if session.KnowledgeBaseID != "" {
// Use session's knowledge base as fallback
agentConfig.KnowledgeBases = []string{session.KnowledgeBaseID}
logger.Infof(ctx, "Using session's knowledge base for agent: %s", session.KnowledgeBaseID)
} else {
// Default to all knowledge bases under the tenant
logger.Infof(ctx, "No knowledge bases specified, fetching all knowledge bases for tenant")
allKBs, err := s.knowledgeBaseService.ListKnowledgeBases(ctx)
if err != nil {
logger.Errorf(ctx, "Failed to list knowledge bases for tenant: %v", err)
return fmt.Errorf("failed to list knowledge bases: %w", err)
}
if len(allKBs) == 0 {
logger.Warnf(ctx, "No knowledge bases available for agent session: %s", sessionID)
return errors.New("no knowledge bases available for agent")
}
// Extract knowledge base IDs
agentConfig.KnowledgeBases = make([]string, len(allKBs))
for i, kb := range allKBs {
if kb == nil {
continue
}
agentConfig.KnowledgeBases[i] = kb.ID
}
logger.Infof(ctx, "Agent defaulting to all %d knowledge base(s) in tenant: %v",
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
// Allow running without knowledge bases (Pure Agent mode)
logger.Infof(ctx, "No knowledge bases specified for agent, running in pure agent mode")
}
} else if len(agentConfig.KnowledgeIDs) > 0 && len(agentConfig.KnowledgeBases) == 0 {
// User specified individual files but no KBs - don't auto-add all KBs
logger.Infof(ctx, "Agent configured with %d individual knowledge ID(s), no KB auto-expansion",
len(agentConfig.KnowledgeIDs))
} else {
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
}
// Build search targets for agent (pre-compute once to avoid repeated queries)
searchTargets, err := s.buildSearchTargets(ctx, tenantInfo.ID, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs)
if err != nil {
logger.Warnf(ctx, "Failed to build search targets for agent: %v", err)
// Continue without search targets, the tool will handle empty targets
}
agentConfig.SearchTargets = searchTargets
logger.Infof(ctx, "Agent search targets built: %d targets", len(searchTargets))
summaryModelID := session.SummaryModelID
if summaryModelID == "" && tenantInfo.ConversationConfig != nil {
summaryModelID = tenantInfo.ConversationConfig.SummaryModelID