feat: 支持输入框内选择知识库和文件,优化选择交互体验
This commit is contained in:
@@ -292,3 +292,58 @@ func (r *knowledgeRepository) CountKnowledgeByStatus(
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// SearchKnowledge searches knowledge items by keyword across the tenant
|
||||
// If keyword is empty, returns recent files
|
||||
// Only returns documents from document-type knowledge bases (excludes FAQ)
|
||||
// Returns (results, hasMore, error)
|
||||
func (r *knowledgeRepository) SearchKnowledge(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
keyword string,
|
||||
offset, limit int,
|
||||
) ([]*types.Knowledge, bool, error) {
|
||||
// Use raw query to properly map knowledge_base_name
|
||||
type KnowledgeWithKBName struct {
|
||||
types.Knowledge
|
||||
KnowledgeBaseName string `gorm:"column:knowledge_base_name"`
|
||||
}
|
||||
|
||||
var results []KnowledgeWithKBName
|
||||
query := r.db.WithContext(ctx).
|
||||
Table("knowledges").
|
||||
Select("knowledges.*, knowledge_bases.name as knowledge_base_name").
|
||||
Joins("JOIN knowledge_bases ON knowledge_bases.id = knowledges.knowledge_base_id").
|
||||
Where("knowledges.tenant_id = ?", tenantID).
|
||||
Where("knowledge_bases.type = ?", types.KnowledgeBaseTypeDocument).
|
||||
Where("knowledges.deleted_at IS NULL")
|
||||
|
||||
// If keyword is provided, filter by file_name or title
|
||||
if keyword != "" {
|
||||
query = query.Where("knowledges.file_name LIKE ? ", "%"+keyword+"%")
|
||||
}
|
||||
|
||||
// Fetch limit+1 to check if there are more results
|
||||
err := query.Order("knowledges.created_at DESC").
|
||||
Offset(offset).
|
||||
Limit(limit + 1).
|
||||
Scan(&results).Error
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// Check if there are more results
|
||||
hasMore := len(results) > limit
|
||||
if hasMore {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
// Convert to []*types.Knowledge
|
||||
knowledges := make([]*types.Knowledge, len(results))
|
||||
for i, r := range results {
|
||||
k := r.Knowledge
|
||||
k.KnowledgeBaseName = r.KnowledgeBaseName
|
||||
knowledges[i] = &k
|
||||
}
|
||||
return knowledges, hasMore, nil
|
||||
}
|
||||
|
||||
@@ -38,6 +38,18 @@ func (r *knowledgeBaseRepository) GetKnowledgeBaseByID(ctx context.Context, id s
|
||||
return &kb, nil
|
||||
}
|
||||
|
||||
// GetKnowledgeBaseByIDs gets knowledge bases by multiple ids
|
||||
func (r *knowledgeBaseRepository) GetKnowledgeBaseByIDs(ctx context.Context, ids []string) ([]*types.KnowledgeBase, error) {
|
||||
if len(ids) == 0 {
|
||||
return []*types.KnowledgeBase{}, nil
|
||||
}
|
||||
var kbs []*types.KnowledgeBase
|
||||
if err := r.db.WithContext(ctx).Where("id IN ?", ids).Find(&kbs).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return kbs, nil
|
||||
}
|
||||
|
||||
// ListKnowledgeBases lists all knowledge bases
|
||||
func (r *knowledgeBaseRepository) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) {
|
||||
var kbs []*types.KnowledgeBase
|
||||
|
||||
@@ -355,10 +355,16 @@ func (e *elasticsearchRepository) deleteByFieldList(ctx context.Context, field s
|
||||
|
||||
// getBaseConds Construct base Elasticsearch query conditions based on retrieval parameters
|
||||
// It creates MUST conditions for required fields and MUST_NOT conditions for excluded fields
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic (search specific documents within knowledge bases)
|
||||
// Returns a JSON string representing the query conditions
|
||||
func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) string {
|
||||
// Build MUST conditions (positive filters)
|
||||
must := make([]map[string]interface{}, 0)
|
||||
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
|
||||
// - If only KnowledgeBaseIDs: search entire knowledge bases
|
||||
// - If only KnowledgeIDs: search specific documents
|
||||
// - If both: search specific documents within the knowledge bases (AND)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
@@ -366,6 +372,13 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
},
|
||||
})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"knowledge_id.keyword": params.KnowledgeIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Build MUST_NOT conditions (negative filters)
|
||||
mustNot := make([]map[string]interface{}, 0)
|
||||
|
||||
@@ -236,8 +236,14 @@ func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
|
||||
// getBaseConds creates the base query conditions for retrieval operations
|
||||
// Returns a slice of Query objects with must and must_not conditions
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic (search specific documents within knowledge bases)
|
||||
func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) []types.Query {
|
||||
must := []types.Query{}
|
||||
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
|
||||
// - If only KnowledgeBaseIDs: search entire knowledge bases
|
||||
// - If only KnowledgeIDs: search specific documents
|
||||
// - If both: search specific documents within the knowledge bases (AND)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
@@ -245,6 +251,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
},
|
||||
}})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"knowledge_id.keyword": params.KnowledgeIDs,
|
||||
},
|
||||
}})
|
||||
}
|
||||
|
||||
mustNot := make([]types.Query, 0)
|
||||
// Exclude disabled chunks (is_enabled = false)
|
||||
// Note: Historical data without is_enabled field will be included (not matching must_not)
|
||||
|
||||
@@ -166,14 +166,25 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
|
||||
conds := make([]clause.Expression, 0)
|
||||
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
|
||||
// - If only KnowledgeBaseIDs: search entire knowledge bases
|
||||
// - If only KnowledgeIDs: search specific documents
|
||||
// - If both: search specific documents within the knowledge bases (AND)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs)
|
||||
// Use standard SQL IN clause instead of @@@ operator for better performance with B-tree index
|
||||
conds = append(conds, clause.IN{
|
||||
Column: "knowledge_base_id",
|
||||
Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs),
|
||||
})
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge IDs: %v", params.KnowledgeIDs)
|
||||
conds = append(conds, clause.IN{
|
||||
Column: "knowledge_id",
|
||||
Values: common.ToInterfaceSlice(params.KnowledgeIDs),
|
||||
})
|
||||
}
|
||||
conds = append(conds, clause.Expr{
|
||||
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
|
||||
Vars: []interface{}{params.Query},
|
||||
@@ -250,13 +261,15 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
whereParts = append(whereParts, fmt.Sprintf("dimension = $%d", len(allVars)+1))
|
||||
allVars = append(allVars, dimension)
|
||||
|
||||
// Knowledge base filter
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
|
||||
// - If only KnowledgeBaseIDs: search entire knowledge bases
|
||||
// - If only KnowledgeIDs: search specific documents
|
||||
// - If both: search specific documents within the knowledge bases (AND)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf(
|
||||
"[Postgres] Filtering vector search by knowledge base IDs: %v",
|
||||
params.KnowledgeBaseIDs,
|
||||
)
|
||||
// Build IN clause with proper placeholders
|
||||
placeholders := make([]string, len(params.KnowledgeBaseIDs))
|
||||
paramStart := len(allVars) + 1
|
||||
for i := range params.KnowledgeBaseIDs {
|
||||
@@ -266,6 +279,20 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
whereParts = append(whereParts, fmt.Sprintf("knowledge_base_id IN (%s)",
|
||||
strings.Join(placeholders, ", ")))
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf(
|
||||
"[Postgres] Filtering vector search by knowledge IDs: %v",
|
||||
params.KnowledgeIDs,
|
||||
)
|
||||
placeholders := make([]string, len(params.KnowledgeIDs))
|
||||
paramStart := len(allVars) + 1
|
||||
for i := range params.KnowledgeIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
|
||||
allVars = append(allVars, params.KnowledgeIDs[i])
|
||||
}
|
||||
whereParts = append(whereParts, fmt.Sprintf("knowledge_id IN (%s)",
|
||||
strings.Join(placeholders, ", ")))
|
||||
}
|
||||
|
||||
// is_enabled filter
|
||||
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
|
||||
|
||||
@@ -433,9 +433,16 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi
|
||||
// Only retrieve enabled chunks
|
||||
must = append(must, qdrant.NewMatchBool(fieldIsEnabled, true))
|
||||
|
||||
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
|
||||
// - If only KnowledgeBaseIDs: search entire knowledge bases
|
||||
// - If only KnowledgeIDs: search specific documents
|
||||
// - If both: search specific documents within the knowledge bases (AND)
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeID, params.KnowledgeIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
|
||||
@@ -445,10 +452,12 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...))
|
||||
}
|
||||
|
||||
return &qdrant.Filter{
|
||||
filter := &qdrant.Filter{
|
||||
Must: must,
|
||||
MustNot: mustNot,
|
||||
}
|
||||
|
||||
return filter
|
||||
}
|
||||
|
||||
// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type
|
||||
|
||||
@@ -141,6 +141,13 @@ func (s *agentService) CreateAgentEngine(
|
||||
}
|
||||
}
|
||||
|
||||
// Get selected documents information (user @ mentioned documents)
|
||||
selectedDocs, err := s.getSelectedDocumentInfos(ctx, config.KnowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get selected document details: %v", err)
|
||||
selectedDocs = []*agent.SelectedDocumentInfo{}
|
||||
}
|
||||
|
||||
systemPromptTemplate := ""
|
||||
if config.UseCustomSystemPrompt {
|
||||
systemPromptTemplate = config.ResolveSystemPrompt(config.WebSearchEnabled)
|
||||
@@ -153,6 +160,7 @@ func (s *agentService) CreateAgentEngine(
|
||||
toolRegistry,
|
||||
eventBus,
|
||||
kbInfos,
|
||||
selectedDocs,
|
||||
contextManager,
|
||||
sessionID,
|
||||
systemPromptTemplate,
|
||||
@@ -173,6 +181,29 @@ func (s *agentService) registerTools(
|
||||
) error {
|
||||
// If no specific tools allowed, register default tools
|
||||
allowedTools := tools.DefaultAllowedTools()
|
||||
|
||||
// Filter out knowledge base tools if no knowledge bases or knowledge IDs are configured
|
||||
hasKnowledge := len(config.KnowledgeBases) > 0 || len(config.KnowledgeIDs) > 0
|
||||
if !hasKnowledge {
|
||||
filteredTools := make([]string, 0)
|
||||
kbTools := map[string]bool{
|
||||
"knowledge_search": true,
|
||||
"grep_chunks": true,
|
||||
"list_knowledge_chunks": true,
|
||||
"query_knowledge_graph": true,
|
||||
"get_document_info": true,
|
||||
"database_query": true,
|
||||
}
|
||||
|
||||
for _, toolName := range allowedTools {
|
||||
if !kbTools[toolName] {
|
||||
filteredTools = append(filteredTools, toolName)
|
||||
}
|
||||
}
|
||||
allowedTools = filteredTools
|
||||
logger.Infof(ctx, "Pure Agent Mode: Knowledge base tools disabled due to empty configuration")
|
||||
}
|
||||
|
||||
// If web search is enabled, add web_search to allowedTools
|
||||
if config.WebSearchEnabled {
|
||||
allowedTools = append(allowedTools, "web_search")
|
||||
@@ -203,16 +234,17 @@ func (s *agentService) registerTools(
|
||||
registry.RegisterTool(
|
||||
tools.NewKnowledgeSearchTool(
|
||||
s.knowledgeBaseService,
|
||||
s.knowledgeService,
|
||||
s.chunkService,
|
||||
tenantID,
|
||||
config.KnowledgeBases,
|
||||
config.SearchTargets,
|
||||
rerankModel,
|
||||
chatModel,
|
||||
s.cfg,
|
||||
))
|
||||
case "grep_chunks":
|
||||
registry.RegisterTool(tools.NewGrepChunksTool(s.db, tenantID, config.KnowledgeBases))
|
||||
logger.Infof(ctx, "Registered grep_chunks tool for tenant: %d", tenantID)
|
||||
registry.RegisterTool(tools.NewGrepChunksTool(s.db, tenantID, config.KnowledgeBases, config.KnowledgeIDs))
|
||||
logger.Infof(ctx, "Registered grep_chunks tool for tenant: %d, KBs: %d, KnowledgeIDs: %d", tenantID, len(config.KnowledgeBases), len(config.KnowledgeIDs))
|
||||
case "list_knowledge_chunks":
|
||||
registry.RegisterTool(tools.NewListKnowledgeChunksTool(tenantID, s.knowledgeService, s.chunkService))
|
||||
case "query_knowledge_graph":
|
||||
@@ -372,3 +404,54 @@ func (s *agentService) getKnowledgeBaseInfos(ctx context.Context, kbIDs []string
|
||||
|
||||
return kbInfos, nil
|
||||
}
|
||||
|
||||
// getSelectedDocumentInfos retrieves detailed information for user-selected documents (via @ mention)
|
||||
// This loads the actual content of the documents to include in the system prompt
|
||||
func (s *agentService) getSelectedDocumentInfos(ctx context.Context, knowledgeIDs []string) ([]*agent.SelectedDocumentInfo, error) {
|
||||
if len(knowledgeIDs) == 0 {
|
||||
return []*agent.SelectedDocumentInfo{}, nil
|
||||
}
|
||||
|
||||
// Get tenant ID from context
|
||||
tenantID := uint64(0)
|
||||
if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok {
|
||||
tenantID = tid
|
||||
}
|
||||
|
||||
// Fetch knowledge metadata
|
||||
knowledges, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get knowledge batch: %w", err)
|
||||
}
|
||||
|
||||
// Build map for quick lookup
|
||||
knowledgeMap := make(map[string]*types.Knowledge)
|
||||
for _, k := range knowledges {
|
||||
if k != nil {
|
||||
knowledgeMap[k.ID] = k
|
||||
}
|
||||
}
|
||||
|
||||
selectedDocs := make([]*agent.SelectedDocumentInfo, 0, len(knowledgeIDs))
|
||||
|
||||
for _, kid := range knowledgeIDs {
|
||||
k, ok := knowledgeMap[kid]
|
||||
if !ok {
|
||||
logger.Warnf(ctx, "Selected knowledge %s not found", kid)
|
||||
continue
|
||||
}
|
||||
|
||||
docInfo := &agent.SelectedDocumentInfo{
|
||||
KnowledgeID: k.ID,
|
||||
KnowledgeBaseID: k.KnowledgeBaseID,
|
||||
Title: k.Title,
|
||||
FileName: k.FileName,
|
||||
FileType: k.FileType,
|
||||
}
|
||||
|
||||
selectedDocs = append(selectedDocs, docInfo)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Loaded %d selected documents metadata for prompt", len(selectedDocs))
|
||||
return selectedDocs, nil
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ type PluginExtractEntity struct {
|
||||
modelService interfaces.ModelService // Model service for calling large language models
|
||||
template *types.PromptTemplateStructured // Template for generating prompts
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
}
|
||||
|
||||
// NewPluginRewrite creates a new query rewriting plugin instance
|
||||
@@ -30,12 +31,14 @@ func NewPluginExtractEntity(
|
||||
eventManager *EventManager,
|
||||
modelService interfaces.ModelService,
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
config *config.Config,
|
||||
) *PluginExtractEntity {
|
||||
res := &PluginExtractEntity{
|
||||
modelService: modelService,
|
||||
template: config.ExtractManager.ExtractEntity,
|
||||
knowledgeBaseRepo: knowledgeBaseRepo,
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
}
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
@@ -65,16 +68,68 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
kb, err := p.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chatManage.KnowledgeBaseID)
|
||||
// Collect all knowledge base IDs to query
|
||||
kbIDSet := make(map[string]struct{})
|
||||
for _, id := range chatManage.KnowledgeBaseIDs {
|
||||
kbIDSet[id] = struct{}{}
|
||||
}
|
||||
|
||||
// If KnowledgeIDs is specified, retrieve them and collect their knowledge base IDs
|
||||
// Also build a mapping from KnowledgeID to KnowledgeBaseID
|
||||
knowledgeToKBMap := make(map[string]string)
|
||||
if len(chatManage.KnowledgeIDs) > 0 {
|
||||
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, chatManage.TenantID, chatManage.KnowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledges: %v", err)
|
||||
return next()
|
||||
}
|
||||
for _, k := range knowledges {
|
||||
kbIDSet[k.KnowledgeBaseID] = struct{}{}
|
||||
knowledgeToKBMap[k.ID] = k.KnowledgeBaseID
|
||||
}
|
||||
}
|
||||
|
||||
// Convert set to slice
|
||||
allKBIDs := make([]string, 0, len(kbIDSet))
|
||||
for id := range kbIDSet {
|
||||
allKBIDs = append(allKBIDs, id)
|
||||
}
|
||||
|
||||
// Batch retrieve all knowledge bases
|
||||
kbs, err := p.knowledgeBaseRepo.GetKnowledgeBaseByIDs(ctx, allKBIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
|
||||
logger.Errorf(ctx, "failed to get knowledge bases: %v", err)
|
||||
return next()
|
||||
}
|
||||
if kb.ExtractConfig == nil {
|
||||
logger.Warnf(ctx, "failed to get extract config")
|
||||
|
||||
// Check if any knowledge base has ExtractConfig enabled and collect their IDs
|
||||
enabledKBSet := make(map[string]struct{})
|
||||
for _, kb := range kbs {
|
||||
if kb.ExtractConfig != nil && kb.ExtractConfig.Enabled {
|
||||
enabledKBSet[kb.ID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(enabledKBSet) == 0 {
|
||||
logger.Debugf(ctx, "no knowledge base has extract config enabled")
|
||||
return next()
|
||||
}
|
||||
|
||||
// Save enabled knowledge base IDs for later use in search_entity
|
||||
enabledKBIDs := make([]string, 0, len(enabledKBSet))
|
||||
for id := range enabledKBSet {
|
||||
enabledKBIDs = append(enabledKBIDs, id)
|
||||
}
|
||||
chatManage.EntityKBIDs = enabledKBIDs
|
||||
|
||||
// Filter knowledgeToKBMap to only include files from enabled knowledge bases
|
||||
entityKnowledge := make(map[string]string)
|
||||
for knowledgeID, kbID := range knowledgeToKBMap {
|
||||
if _, ok := enabledKBSet[kbID]; ok {
|
||||
entityKnowledge[knowledgeID] = kbID
|
||||
}
|
||||
}
|
||||
chatManage.EntityKnowledge = entityKnowledge
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: p.template.Description,
|
||||
Examples: p.template.Examples,
|
||||
|
||||
@@ -66,35 +66,54 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
return ErrGetRerankModel.WithError(err)
|
||||
}
|
||||
|
||||
// Prepare passages for reranking
|
||||
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
|
||||
"candidate_cnt": len(chatManage.SearchResult),
|
||||
})
|
||||
// Prepare passages for reranking (excluding DirectLoad results)
|
||||
var passages []string
|
||||
var candidatesToRerank []*types.SearchResult
|
||||
var directLoadResults []*types.SearchResult
|
||||
|
||||
for _, result := range chatManage.SearchResult {
|
||||
if result.MatchType == types.MatchTypeDirectLoad {
|
||||
directLoadResults = append(directLoadResults, result)
|
||||
pipelineInfo(ctx, "Rerank", "direct_load_skip", map[string]interface{}{
|
||||
"chunk_id": result.ID,
|
||||
})
|
||||
continue
|
||||
}
|
||||
// 合并Content和ImageInfo的文本内容
|
||||
passage := getEnrichedPassage(ctx, result)
|
||||
passages = append(passages, passage)
|
||||
candidatesToRerank = append(candidatesToRerank, result)
|
||||
}
|
||||
|
||||
// Single rerank call with RewriteQuery, use threshold degradation if no results
|
||||
originalThreshold := chatManage.RerankThreshold
|
||||
rerankResp := p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages)
|
||||
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
|
||||
"total_cnt": len(chatManage.SearchResult),
|
||||
"candidate_cnt": len(candidatesToRerank),
|
||||
"direct_cnt": len(directLoadResults),
|
||||
})
|
||||
|
||||
// If no results and threshold is high enough, try with lower threshold
|
||||
if len(rerankResp) == 0 && originalThreshold > 0.3 {
|
||||
degradedThreshold := originalThreshold * 0.7
|
||||
if degradedThreshold < 0.3 {
|
||||
degradedThreshold = 0.3
|
||||
var rerankResp []rerank.RankResult
|
||||
|
||||
// Only call rerank model if there are candidates
|
||||
if len(candidatesToRerank) > 0 {
|
||||
// Single rerank call with RewriteQuery, use threshold degradation if no results
|
||||
originalThreshold := chatManage.RerankThreshold
|
||||
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)
|
||||
|
||||
// If no results and threshold is high enough, try with lower threshold
|
||||
if len(rerankResp) == 0 && originalThreshold > 0.3 {
|
||||
degradedThreshold := originalThreshold * 0.7
|
||||
if degradedThreshold < 0.3 {
|
||||
degradedThreshold = 0.3
|
||||
}
|
||||
pipelineInfo(ctx, "Rerank", "threshold_degrade", map[string]interface{}{
|
||||
"original": originalThreshold,
|
||||
"degraded": degradedThreshold,
|
||||
})
|
||||
chatManage.RerankThreshold = degradedThreshold
|
||||
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages, candidatesToRerank)
|
||||
// Restore original threshold
|
||||
chatManage.RerankThreshold = originalThreshold
|
||||
}
|
||||
pipelineInfo(ctx, "Rerank", "threshold_degrade", map[string]interface{}{
|
||||
"original": originalThreshold,
|
||||
"degraded": degradedThreshold,
|
||||
})
|
||||
chatManage.RerankThreshold = degradedThreshold
|
||||
rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages)
|
||||
// Restore original threshold
|
||||
chatManage.RerankThreshold = originalThreshold
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
|
||||
@@ -114,9 +133,14 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
for i := range chatManage.SearchResult {
|
||||
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
|
||||
}
|
||||
reranked := make([]*types.SearchResult, 0, len(rerankResp))
|
||||
reranked := make([]*types.SearchResult, 0, len(rerankResp)+len(directLoadResults))
|
||||
|
||||
// Process reranked results
|
||||
for _, rr := range rerankResp {
|
||||
sr := chatManage.SearchResult[rr.Index]
|
||||
if rr.Index >= len(candidatesToRerank) {
|
||||
continue
|
||||
}
|
||||
sr := candidatesToRerank[rr.Index]
|
||||
base := sr.Score
|
||||
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
|
||||
modelScore := rr.RelevanceScore
|
||||
@@ -130,6 +154,23 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
})
|
||||
reranked = append(reranked, sr)
|
||||
}
|
||||
|
||||
// Process direct load results (bypass rerank model, assume high relevance)
|
||||
for _, sr := range directLoadResults {
|
||||
base := sr.Score
|
||||
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
|
||||
// Assign high model score for direct load items
|
||||
modelScore := 1.0
|
||||
sr.Score = compositeScore(sr, modelScore, base)
|
||||
pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{
|
||||
"chunk_id": sr.ID,
|
||||
"base_score": fmt.Sprintf("%.4f", base),
|
||||
"model_score": fmt.Sprintf("%.4f", modelScore),
|
||||
"final_score": fmt.Sprintf("%.4f", sr.Score),
|
||||
"match_type": sr.MatchType,
|
||||
})
|
||||
reranked = append(reranked, sr)
|
||||
}
|
||||
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
|
||||
chatManage.RerankResult = final
|
||||
|
||||
@@ -160,6 +201,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
// rerank performs the actual reranking operation with given query and passages
|
||||
func (p *PluginRerank) rerank(ctx context.Context,
|
||||
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
|
||||
candidates []*types.SearchResult,
|
||||
) []rerank.RankResult {
|
||||
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
|
||||
"query_variant": query,
|
||||
@@ -179,21 +221,26 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
||||
"threshold": chatManage.RerankThreshold,
|
||||
})
|
||||
for i := range min(5, len(rerankResp)) {
|
||||
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
|
||||
"rank": i + 1,
|
||||
"score": rerankResp[i].RelevanceScore,
|
||||
"chunk_id": chatManage.SearchResult[rerankResp[i].Index].ID,
|
||||
"match_type": chatManage.SearchResult[rerankResp[i].Index].MatchType,
|
||||
"chunk_type": chatManage.SearchResult[rerankResp[i].Index].ChunkType,
|
||||
"content": chatManage.SearchResult[rerankResp[i].Index].Content,
|
||||
})
|
||||
if rerankResp[i].Index < len(candidates) {
|
||||
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
|
||||
"rank": i + 1,
|
||||
"score": rerankResp[i].RelevanceScore,
|
||||
"chunk_id": candidates[rerankResp[i].Index].ID,
|
||||
"match_type": candidates[rerankResp[i].Index].MatchType,
|
||||
"chunk_type": candidates[rerankResp[i].Index].ChunkType,
|
||||
"content": candidates[rerankResp[i].Index].Content,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Filter results based on threshold with special handling for history matches
|
||||
rankFilter := []rerank.RankResult{}
|
||||
for _, result := range rerankResp {
|
||||
if result.Index >= len(candidates) {
|
||||
continue
|
||||
}
|
||||
th := chatManage.RerankThreshold
|
||||
matchType := chatManage.SearchResult[result.Index].MatchType
|
||||
matchType := candidates[result.Index].MatchType
|
||||
if matchType == types.MatchTypeHistory {
|
||||
th = math.Max(th-0.1, 0.5) // Lower threshold for history matches
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
type PluginSearch struct {
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService
|
||||
knowledgeService interfaces.KnowledgeService
|
||||
chunkService interfaces.ChunkService
|
||||
config *config.Config
|
||||
webSearchService interfaces.WebSearchService
|
||||
tenantService interfaces.TenantService
|
||||
@@ -28,6 +29,7 @@ type PluginSearch struct {
|
||||
func NewPluginSearch(eventManager *EventManager,
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
chunkService interfaces.ChunkService,
|
||||
config *config.Config,
|
||||
webSearchService interfaces.WebSearchService,
|
||||
tenantService interfaces.TenantService,
|
||||
@@ -36,6 +38,7 @@ func NewPluginSearch(eventManager *EventManager,
|
||||
res := &PluginSearch{
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
chunkService: chunkService,
|
||||
config: config,
|
||||
webSearchService: webSearchService,
|
||||
tenantService: tenantService,
|
||||
@@ -54,35 +57,25 @@ func (p *PluginSearch) ActivationEvents() []types.EventType {
|
||||
func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
// Get knowledge base IDs list
|
||||
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
|
||||
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
|
||||
// Fall back to single knowledge base
|
||||
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
|
||||
pipelineInfo(ctx, "Search", "fallback_kb", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"kb_id": chatManage.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
|
||||
if len(knowledgeBaseIDs) == 0 {
|
||||
// Check if we have search targets
|
||||
if len(chatManage.SearchTargets) == 0 && len(chatManage.KnowledgeBaseIDs) == 0 && len(chatManage.KnowledgeIDs) == 0 {
|
||||
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return ErrSearch.WithError(nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"kb_ids": strings.Join(knowledgeBaseIDs, ","),
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"web_enabled": chatManage.WebSearchEnabled,
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"search_targets": len(chatManage.SearchTargets),
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"web_enabled": chatManage.WebSearchEnabled,
|
||||
})
|
||||
|
||||
// Run KB search and web search concurrently
|
||||
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
|
||||
"kb_count": len(knowledgeBaseIDs),
|
||||
"search_targets": len(chatManage.SearchTargets),
|
||||
"embedding_top_k": chatManage.EmbeddingTopK,
|
||||
"vector_threshold": chatManage.VectorThreshold,
|
||||
"keyword_threshold": chatManage.KeywordThreshold,
|
||||
@@ -92,10 +85,10 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
allResults := make([]*types.SearchResult, 0)
|
||||
|
||||
wg.Add(2)
|
||||
// Goroutine 1: Knowledge base search (rewrite + processed)
|
||||
// Goroutine 1: Knowledge base search using SearchTargets
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
kbResults := p.searchKnowledgeBases(ctx, knowledgeBaseIDs, chatManage)
|
||||
kbResults := p.searchByTargets(ctx, chatManage)
|
||||
if len(kbResults) > 0 {
|
||||
mu.Lock()
|
||||
allResults = append(allResults, kbResults...)
|
||||
@@ -141,11 +134,11 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
})
|
||||
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
|
||||
expKwTh := chatManage.KeywordThreshold * 0.8
|
||||
// Concurrent expansion retrieval across queries and KBs
|
||||
// Concurrent expansion retrieval across queries and search targets
|
||||
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
|
||||
var muExp sync.Mutex
|
||||
var wgExp sync.WaitGroup
|
||||
jobs := len(expansions) * len(knowledgeBaseIDs)
|
||||
jobs := len(expansions) * len(chatManage.SearchTargets)
|
||||
capSem := 16
|
||||
if jobs < capSem {
|
||||
capSem = jobs
|
||||
@@ -159,9 +152,9 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
"cap": capSem,
|
||||
})
|
||||
for _, q := range expansions {
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
for _, target := range chatManage.SearchTargets {
|
||||
wgExp.Add(1)
|
||||
go func(q string, kbID string) {
|
||||
go func(q string, t *types.SearchTarget) {
|
||||
defer wgExp.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
@@ -173,17 +166,21 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
DisableVectorMatch: true,
|
||||
DisableKeywordsMatch: false,
|
||||
}
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, kbID, paramsExp)
|
||||
// Apply knowledge ID filter if this is a partial KB search
|
||||
if t.Type == types.SearchTargetTypeKnowledge {
|
||||
paramsExp.KnowledgeIDs = t.KnowledgeIDs
|
||||
}
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, paramsExp)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
|
||||
"kb_id": kbID,
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
if len(res) > 0 {
|
||||
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
|
||||
"kb_id": kbID,
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"query": q,
|
||||
"hits": len(res),
|
||||
})
|
||||
@@ -191,7 +188,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
expResults = append(expResults, res...)
|
||||
muExp.Unlock()
|
||||
}
|
||||
}(q, kbID)
|
||||
}(q, target)
|
||||
}
|
||||
}
|
||||
wgExp.Wait()
|
||||
@@ -308,46 +305,89 @@ func buildContentSignature(content string) string {
|
||||
return searchutil.BuildContentSignature(content)
|
||||
}
|
||||
|
||||
// searchKnowledgeBases performs KB searches across KB IDs using RewriteQuery only
|
||||
func (p *PluginSearch) searchKnowledgeBases(
|
||||
// searchByTargets performs KB searches using pre-computed SearchTargets
|
||||
// This is the main search method that uses the unified search targets
|
||||
func (p *PluginSearch) searchByTargets(
|
||||
ctx context.Context,
|
||||
knowledgeBaseIDs []string,
|
||||
chatManage *types.ChatManage,
|
||||
) []*types.SearchResult {
|
||||
// Build params for rewrite query
|
||||
baseParams := types.SearchParams{
|
||||
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
||||
VectorThreshold: chatManage.VectorThreshold,
|
||||
KeywordThreshold: chatManage.KeywordThreshold,
|
||||
MatchCount: chatManage.EmbeddingTopK,
|
||||
if len(chatManage.SearchTargets) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var results []*types.SearchResult
|
||||
|
||||
// Search with rewrite query only (removed duplicate ProcessedQuery search)
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
// Search each target concurrently
|
||||
for _, target := range chatManage.SearchTargets {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
go func(t *types.SearchTarget) {
|
||||
defer wg.Done()
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams)
|
||||
|
||||
// List of knowledge IDs to perform vector search on
|
||||
// Default to all IDs in the target
|
||||
searchKnowledgeIDs := t.KnowledgeIDs
|
||||
|
||||
// Try direct loading for specific knowledge targets
|
||||
if t.Type == types.SearchTargetTypeKnowledge {
|
||||
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
|
||||
|
||||
if len(directResults) > 0 {
|
||||
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"loaded_count": len(directResults),
|
||||
"skipped_ids": len(skippedIDs),
|
||||
})
|
||||
mu.Lock()
|
||||
results = append(results, directResults...)
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
// If all files were loaded directly, we don't need to search anything
|
||||
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Otherwise, only search the files that were skipped (too large)
|
||||
searchKnowledgeIDs = skippedIDs
|
||||
}
|
||||
|
||||
// If no IDs left to search (and we are in Knowledge mode), we are done
|
||||
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Build params for rewrite query
|
||||
params := types.SearchParams{
|
||||
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
||||
VectorThreshold: chatManage.VectorThreshold,
|
||||
KeywordThreshold: chatManage.KeywordThreshold,
|
||||
MatchCount: chatManage.EmbeddingTopK,
|
||||
}
|
||||
// Apply knowledge ID filter if this is a partial KB search
|
||||
if t.Type == types.SearchTargetTypeKnowledge {
|
||||
params.KnowledgeIDs = searchKnowledgeIDs
|
||||
}
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"query": baseParams.QueryText,
|
||||
"error": err.Error(),
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"target_type": t.Type,
|
||||
"query": params.QueryText,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"hit_count": len(res),
|
||||
"kb_id": t.KnowledgeBaseID,
|
||||
"target_type": t.Type,
|
||||
"hit_count": len(res),
|
||||
})
|
||||
mu.Lock()
|
||||
results = append(results, res...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
}(target)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
@@ -358,6 +398,93 @@ func (p *PluginSearch) searchKnowledgeBases(
|
||||
return results
|
||||
}
|
||||
|
||||
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
|
||||
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
|
||||
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
|
||||
if len(knowledgeIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Limit direct loading to avoid OOM or context overflow
|
||||
// 50 chunks * ~500 chars/chunk ~= 25k chars
|
||||
const maxTotalChunks = 50
|
||||
|
||||
var allChunks []*types.Chunk
|
||||
var skippedIDs []string
|
||||
loadedKnowledgeIDs := make(map[string]bool)
|
||||
|
||||
for _, kid := range knowledgeIDs {
|
||||
// Optimization: Check chunk count first if possible?
|
||||
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
|
||||
skippedIDs = append(skippedIDs, kid)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(allChunks)+len(chunks) > maxTotalChunks {
|
||||
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
|
||||
kid, len(allChunks), len(chunks), maxTotalChunks)
|
||||
skippedIDs = append(skippedIDs, kid)
|
||||
continue
|
||||
}
|
||||
allChunks = append(allChunks, chunks...)
|
||||
loadedKnowledgeIDs[kid] = true
|
||||
}
|
||||
|
||||
if len(allChunks) == 0 {
|
||||
return nil, skippedIDs
|
||||
}
|
||||
|
||||
// Fetch Knowledge metadata
|
||||
var uniqueKIDs []string
|
||||
for kid := range loadedKnowledgeIDs {
|
||||
uniqueKIDs = append(uniqueKIDs, kid)
|
||||
}
|
||||
|
||||
knowledgeMap := make(map[string]*types.Knowledge)
|
||||
if len(uniqueKIDs) > 0 {
|
||||
knowledges, err := p.knowledgeService.GetKnowledgeBatch(ctx, tenantID, uniqueKIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
|
||||
// Continue without metadata
|
||||
} else {
|
||||
for _, k := range knowledges {
|
||||
knowledgeMap[k.ID] = k
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var results []*types.SearchResult
|
||||
for _, chunk := range allChunks {
|
||||
res := &types.SearchResult{
|
||||
ID: chunk.ID,
|
||||
Content: chunk.Content,
|
||||
Score: 1.0, // Maximum score for direct matches
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
ChunkIndex: chunk.ChunkIndex,
|
||||
MatchType: types.MatchTypeDirectLoad,
|
||||
ChunkType: string(chunk.ChunkType),
|
||||
ParentChunkID: chunk.ParentChunkID,
|
||||
ImageInfo: chunk.ImageInfo,
|
||||
ChunkMetadata: chunk.Metadata,
|
||||
StartAt: chunk.StartAt,
|
||||
EndAt: chunk.EndAt,
|
||||
}
|
||||
|
||||
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
|
||||
res.KnowledgeTitle = k.Title
|
||||
res.KnowledgeFilename = k.FileName
|
||||
res.KnowledgeSource = k.Source
|
||||
res.Metadata = k.GetMetadata()
|
||||
}
|
||||
|
||||
results = append(results, res)
|
||||
}
|
||||
|
||||
return results, skippedIDs
|
||||
}
|
||||
|
||||
// searchWebIfEnabled executes web search when enabled and returns converted results
|
||||
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
||||
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil || chatManage.TenantID <= 0 {
|
||||
|
||||
@@ -47,50 +47,81 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// Get knowledge base IDs list
|
||||
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
|
||||
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
|
||||
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
|
||||
logger.Infof(ctx, "No KnowledgeBaseIDs provided, falling back to single KB: %s", chatManage.KnowledgeBaseID)
|
||||
}
|
||||
// Use EntityKBIDs (knowledge bases with ExtractConfig enabled)
|
||||
knowledgeBaseIDs := chatManage.EntityKBIDs
|
||||
// Use EntityKnowledge (KnowledgeID -> KnowledgeBaseID mapping for graph-enabled files)
|
||||
entityKnowledge := chatManage.EntityKnowledge
|
||||
|
||||
if len(knowledgeBaseIDs) == 0 {
|
||||
logger.Warnf(ctx, "No knowledge base IDs available for entity search")
|
||||
if len(knowledgeBaseIDs) == 0 && len(entityKnowledge) == 0 {
|
||||
logger.Warnf(ctx, "No knowledge base IDs or knowledge IDs with ExtractConfig enabled for entity search")
|
||||
return next()
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
|
||||
|
||||
// Parallel search across multiple knowledge bases
|
||||
// Parallel search across multiple knowledge bases and individual files
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var allNodes []*types.GraphNode
|
||||
var allRelations []*types.GraphRelation
|
||||
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg.Done()
|
||||
// If specific KnowledgeIDs are provided, search by individual files
|
||||
if len(entityKnowledge) > 0 {
|
||||
logger.Infof(ctx, "Searching entities across %d knowledge file(s)", len(entityKnowledge))
|
||||
for knowledgeID, kbID := range entityKnowledge {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID, knowledgeID string) {
|
||||
defer wg.Done()
|
||||
|
||||
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{
|
||||
KnowledgeBase: knowledgeBaseID,
|
||||
Knowledge: knowledgeID,
|
||||
}, entity)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search entity in Knowledge %s: %v", knowledgeID, err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"KB %s entity search result count: %d nodes, %d relations",
|
||||
knowledgeBaseID,
|
||||
len(graph.Node),
|
||||
len(graph.Relation),
|
||||
)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge %s entity search result count: %d nodes, %d relations",
|
||||
knowledgeID,
|
||||
len(graph.Node),
|
||||
len(graph.Relation),
|
||||
)
|
||||
|
||||
mu.Lock()
|
||||
allNodes = append(allNodes, graph.Node...)
|
||||
allRelations = append(allRelations, graph.Relation...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
mu.Lock()
|
||||
allNodes = append(allNodes, graph.Node...)
|
||||
allRelations = append(allRelations, graph.Relation...)
|
||||
mu.Unlock()
|
||||
}(kbID, knowledgeID)
|
||||
}
|
||||
} else {
|
||||
// Otherwise, search by knowledge base
|
||||
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wg.Add(1)
|
||||
go func(knowledgeBaseID string) {
|
||||
defer wg.Done()
|
||||
|
||||
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"KB %s entity search result count: %d nodes, %d relations",
|
||||
knowledgeBaseID,
|
||||
len(graph.Node),
|
||||
len(graph.Relation),
|
||||
)
|
||||
|
||||
mu.Lock()
|
||||
allNodes = append(allNodes, graph.Node...)
|
||||
allRelations = append(allRelations, graph.Relation...)
|
||||
mu.Unlock()
|
||||
}(kbID)
|
||||
}
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
@@ -35,6 +35,7 @@ func NewPluginSearchParallel(
|
||||
eventManager *EventManager,
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
chunkService interfaces.ChunkService,
|
||||
config *config.Config,
|
||||
webSearchService interfaces.WebSearchService,
|
||||
tenantService interfaces.TenantService,
|
||||
@@ -47,6 +48,7 @@ func NewPluginSearchParallel(
|
||||
searchPlugin := &PluginSearch{
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
chunkService: chunkService,
|
||||
config: config,
|
||||
webSearchService: webSearchService,
|
||||
tenantService: tenantService,
|
||||
|
||||
@@ -266,7 +266,6 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
StartTime: time.Now(),
|
||||
},
|
||||
Params: &types.ChatManage{
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
VectorThreshold: e.config.Conversation.VectorThreshold,
|
||||
KeywordThreshold: e.config.Conversation.KeywordThreshold,
|
||||
EmbeddingTopK: e.config.Conversation.EmbeddingTopK,
|
||||
@@ -311,7 +310,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
logger.Info(newCtx, "Evaluation task status set to running")
|
||||
|
||||
// Execute actual evaluation
|
||||
if err := e.EvalDataset(newCtx, detail); err != nil {
|
||||
if err := e.EvalDataset(newCtx, detail, knowledgeBaseID); err != nil {
|
||||
detail.Task.Status = types.EvaluationStatueFailed
|
||||
detail.Task.ErrMsg = err.Error()
|
||||
logger.Errorf(newCtx, "Evaluation task failed: %v, task ID: %s", err, taskID)
|
||||
@@ -329,7 +328,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
|
||||
// EvalDataset performs the actual evaluation of a dataset
|
||||
// Processes each QA pair in parallel and records metrics
|
||||
func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.EvaluationDetail) error {
|
||||
func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.EvaluationDetail, knowledgeBaseID string) error {
|
||||
logger.Info(ctx, "Start evaluating dataset")
|
||||
logger.Infof(ctx, "Task ID: %s, Dataset ID: %s", detail.Task.ID, detail.Task.DatasetID)
|
||||
|
||||
@@ -352,7 +351,7 @@ func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.Evalu
|
||||
logger.Infof(ctx, "Creating knowledge from %d passages", len(passages))
|
||||
|
||||
// Create knowledge base from passages
|
||||
knowledge, err := e.knowledgeService.CreateKnowledgeFromPassage(ctx, detail.Params.KnowledgeBaseID, passages)
|
||||
knowledge, err := e.knowledgeService.CreateKnowledgeFromPassage(ctx, knowledgeBaseID, passages)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create knowledge from passages: %v", err)
|
||||
return err
|
||||
@@ -366,12 +365,12 @@ func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.Evalu
|
||||
logger.Errorf(ctx, "Failed to delete knowledge: %v, knowledge ID: %s", err, knowledge.ID)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Cleaning up resources - deleting knowledge base: %s", detail.Params.KnowledgeBaseID)
|
||||
if err := e.knowledgeBaseService.DeleteKnowledgeBase(ctx, detail.Params.KnowledgeBaseID); err != nil {
|
||||
logger.Infof(ctx, "Cleaning up resources - deleting knowledge base: %s", knowledgeBaseID)
|
||||
if err := e.knowledgeBaseService.DeleteKnowledgeBase(ctx, knowledgeBaseID); err != nil {
|
||||
logger.Errorf(
|
||||
ctx,
|
||||
"Failed to delete knowledge base: %v, knowledge base ID: %s",
|
||||
err, detail.Params.KnowledgeBaseID,
|
||||
err, knowledgeBaseID,
|
||||
)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -5766,3 +5766,13 @@ func (s *knowledgeService) getOrCreateTagInTarget(
|
||||
logger.Infof(ctx, "Created tag %s (ID: %s) in target KB %s", newTag.Name, newTag.ID, dstKnowledgeBaseID)
|
||||
return newTag.ID
|
||||
}
|
||||
|
||||
// SearchKnowledge searches knowledge items by keyword across the tenant
|
||||
func (s *knowledgeService) SearchKnowledge(ctx context.Context, keyword string, offset, limit int) ([]*types.Knowledge, bool, error) {
|
||||
tenantID, ok := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
if !ok {
|
||||
return nil, false, werrors.NewUnauthorizedError("Tenant ID not found in context")
|
||||
}
|
||||
return s.repo.SearchKnowledge(ctx, tenantID, keyword, offset, limit)
|
||||
}
|
||||
|
||||
|
||||
@@ -491,6 +491,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
TopK: matchCount,
|
||||
Threshold: params.VectorThreshold,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
KnowledgeIDs: params.KnowledgeIDs,
|
||||
}
|
||||
|
||||
// For FAQ knowledge base, use FAQ index
|
||||
@@ -512,6 +513,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
TopK: matchCount,
|
||||
Threshold: params.KeywordThreshold,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
KnowledgeIDs: params.KnowledgeIDs,
|
||||
})
|
||||
logger.Info(ctx, "Keyword retrieval parameters setup completed")
|
||||
}
|
||||
|
||||
@@ -43,6 +43,7 @@ type sessionService struct {
|
||||
agentService interfaces.AgentService // Service for agent operations
|
||||
sessionStorage llmcontext.ContextStorage // Session storage
|
||||
knowledgeService interfaces.KnowledgeService // Service for knowledge operations
|
||||
chunkService interfaces.ChunkService // Service for chunk operations
|
||||
redisClient *redis.Client // Redis client for temp KB state
|
||||
}
|
||||
|
||||
@@ -52,6 +53,7 @@ func NewSessionService(cfg *config.Config,
|
||||
messageRepo interfaces.MessageRepository,
|
||||
knowledgeBaseService interfaces.KnowledgeBaseService,
|
||||
knowledgeService interfaces.KnowledgeService,
|
||||
chunkService interfaces.ChunkService,
|
||||
modelService interfaces.ModelService,
|
||||
tenantService interfaces.TenantService,
|
||||
eventManager *chatpipline.EventManager,
|
||||
@@ -65,6 +67,7 @@ func NewSessionService(cfg *config.Config,
|
||||
messageRepo: messageRepo,
|
||||
knowledgeBaseService: knowledgeBaseService,
|
||||
knowledgeService: knowledgeService,
|
||||
chunkService: chunkService,
|
||||
modelService: modelService,
|
||||
tenantService: tenantService,
|
||||
eventManager: eventManager,
|
||||
@@ -394,6 +397,7 @@ func (s *sessionService) KnowledgeQA(
|
||||
session *types.Session,
|
||||
query string,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
assistantMessageID string,
|
||||
summaryModelID string,
|
||||
webSearchEnabled bool,
|
||||
@@ -414,14 +418,13 @@ func (s *sessionService) KnowledgeQA(
|
||||
logger.Infof(ctx, "No knowledge base IDs provided, using session default: %s", session.KnowledgeBaseID)
|
||||
} else {
|
||||
logger.Warnf(ctx, "Session has no associated knowledge base, session ID: %s", session.ID)
|
||||
return errors.New("session has no knowledge base")
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Using knowledge bases: %v", knowledgeBaseIDs)
|
||||
|
||||
// Determine chat model ID: prioritize request's summaryModelID, then Remote models
|
||||
chatModelID, err := s.selectChatModelIDWithOverride(ctx, session, knowledgeBaseIDs, summaryModelID)
|
||||
chatModelID, err := s.selectChatModelIDWithOverride(ctx, session, knowledgeBaseIDs, knowledgeIDs, summaryModelID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -511,20 +514,29 @@ func (s *sessionService) KnowledgeQA(
|
||||
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
|
||||
}
|
||||
|
||||
// Build unified search targets (computed once, used throughout pipeline)
|
||||
searchTargets, err := s.buildSearchTargets(ctx, session.TenantID, knowledgeBaseIDs, knowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to build search targets: %v", err)
|
||||
}
|
||||
|
||||
// Create chat management object with session settings
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Creating chat manage object, knowledge base IDs: %v, chat model ID: %s",
|
||||
"Creating chat manage object, knowledge base IDs: %v, knowledge IDs: %v, chat model ID: %s, search targets: %d",
|
||||
knowledgeBaseIDs,
|
||||
knowledgeIDs,
|
||||
chatModelID,
|
||||
len(searchTargets),
|
||||
)
|
||||
chatManage := &types.ChatManage{
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
SessionID: session.ID,
|
||||
MessageID: assistantMessageID, // NEW: For event emission in pipeline
|
||||
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
|
||||
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
|
||||
MessageID: assistantMessageID, // NEW: For event emission in pipeline
|
||||
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
|
||||
KnowledgeIDs: knowledgeIDs, // Specific knowledge (file) IDs
|
||||
SearchTargets: searchTargets, // Pre-computed search targets
|
||||
VectorThreshold: vectorThreshold,
|
||||
KeywordThreshold: keywordThreshold,
|
||||
EmbeddingTopK: embeddingTopK,
|
||||
@@ -546,9 +558,22 @@ func (s *sessionService) KnowledgeQA(
|
||||
EnableQueryExpansion: enableQueryExpansion,
|
||||
}
|
||||
|
||||
// Determine pipeline based on knowledge bases availability
|
||||
// If no knowledge bases are selected, use pure chat pipeline
|
||||
var pipeline []types.EventType
|
||||
if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) == 0 {
|
||||
logger.Info(ctx, "No knowledge bases selected, using chat_stream pipeline")
|
||||
pipeline = types.Pipline["chat_stream"]
|
||||
// For pure chat, UserContent is the Query (since INTO_CHAT_MESSAGE is skipped)
|
||||
chatManage.UserContent = query
|
||||
} else {
|
||||
logger.Info(ctx, "Knowledge bases selected, using rag_stream pipeline")
|
||||
pipeline = types.Pipline["rag_stream"]
|
||||
}
|
||||
|
||||
// Start knowledge QA event processing
|
||||
logger.Info(ctx, "Triggering knowledge base question answering event")
|
||||
err = s.KnowledgeQAByEvent(ctx, chatManage, types.Pipline["rag_stream"])
|
||||
logger.Info(ctx, "Triggering question answering event")
|
||||
err = s.KnowledgeQAByEvent(ctx, chatManage, pipeline)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"session_id": session.ID,
|
||||
@@ -592,6 +617,7 @@ func (s *sessionService) selectChatModelIDWithOverride(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
summaryModelID string,
|
||||
) (string, error) {
|
||||
// First, check if request has summaryModelID override
|
||||
@@ -612,33 +638,47 @@ func (s *sessionService) selectChatModelIDWithOverride(
|
||||
}
|
||||
|
||||
// If no valid override, use default selection logic
|
||||
return s.selectChatModelID(ctx, session, knowledgeBaseIDs)
|
||||
return s.selectChatModelID(ctx, session, knowledgeBaseIDs, knowledgeIDs)
|
||||
}
|
||||
|
||||
// selectChatModelID selects the appropriate chat model ID with priority for Remote models
|
||||
// Priority order:
|
||||
// 1. Session's SummaryModelID if it's a Remote model
|
||||
// 2. First knowledge base with a Remote model
|
||||
// 2. First knowledge base with a Remote model (from knowledgeBaseIDs or derived from knowledgeIDs)
|
||||
// 3. Session's SummaryModelID (if not Remote)
|
||||
// 4. First knowledge base's SummaryModelID
|
||||
func (s *sessionService) selectChatModelID(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
) (string, error) {
|
||||
|
||||
// First, check if session has a SummaryModelID and if it's a Remote model
|
||||
if session.SummaryModelID != "" {
|
||||
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
|
||||
if err == nil && model != nil && model.Source == types.ModelSourceRemote {
|
||||
logger.Infof(ctx, "Using session's Remote summary model: %s", session.SummaryModelID)
|
||||
return session.SummaryModelID, nil
|
||||
} else if err == nil && model != nil {
|
||||
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+
|
||||
"checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
return session.SummaryModelID, nil
|
||||
}
|
||||
// If no knowledge base IDs but have knowledge IDs, derive KB IDs from knowledge IDs
|
||||
if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) > 0 {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
knowledgeList, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get knowledge batch for model selection: %v", err)
|
||||
} else {
|
||||
// Collect unique KB IDs from knowledge items
|
||||
kbIDSet := make(map[string]bool)
|
||||
for _, k := range knowledgeList {
|
||||
if k != nil && k.KnowledgeBaseID != "" {
|
||||
kbIDSet[k.KnowledgeBaseID] = true
|
||||
}
|
||||
}
|
||||
for kbID := range kbIDSet {
|
||||
knowledgeBaseIDs = append(knowledgeBaseIDs, kbID)
|
||||
}
|
||||
logger.Infof(ctx, "Derived %d knowledge base IDs from %d knowledge IDs for model selection",
|
||||
len(knowledgeBaseIDs), len(knowledgeIDs))
|
||||
}
|
||||
}
|
||||
|
||||
// If no Remote model found from session, check knowledge bases for Remote models
|
||||
if len(knowledgeBaseIDs) > 0 {
|
||||
// Try to find a knowledge base with Remote model
|
||||
@@ -683,12 +723,79 @@ func (s *sessionService) selectChatModelID(
|
||||
}
|
||||
}
|
||||
|
||||
// No knowledge bases - use session's SummaryModelID if available
|
||||
if session.SummaryModelID != "" {
|
||||
logger.Infof(ctx, "No knowledge bases, using session's summary model: %s", session.SummaryModelID)
|
||||
return session.SummaryModelID, nil
|
||||
}
|
||||
|
||||
logger.Error(ctx, "No chat model ID available")
|
||||
return "", errors.New(
|
||||
"no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID",
|
||||
"no chat model ID available: session has no SummaryModelID and no knowledge bases configured",
|
||||
)
|
||||
}
|
||||
|
||||
// buildSearchTargets computes the unified search targets from knowledgeBaseIDs and knowledgeIDs
|
||||
// This is called once at the request entry point to avoid repeated queries later in the pipeline
|
||||
// Logic:
|
||||
// - For each knowledgeBaseID: create a SearchTargetTypeKnowledgeBase target
|
||||
// - For each knowledgeID: find its knowledgeBaseID, if the KB is already in the list, skip (covered by full KB search)
|
||||
// otherwise create a SearchTargetTypeKnowledge target grouped by KB
|
||||
func (s *sessionService) buildSearchTargets(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
knowledgeBaseIDs []string,
|
||||
knowledgeIDs []string,
|
||||
) (types.SearchTargets, error) {
|
||||
var targets types.SearchTargets
|
||||
|
||||
// Track which KBs are fully searched
|
||||
fullKBSet := make(map[string]bool)
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
fullKBSet[kbID] = true
|
||||
targets = append(targets, &types.SearchTarget{
|
||||
Type: types.SearchTargetTypeKnowledgeBase,
|
||||
KnowledgeBaseID: kbID,
|
||||
})
|
||||
}
|
||||
|
||||
// Process individual knowledge IDs
|
||||
if len(knowledgeIDs) > 0 {
|
||||
knowledgeList, err := s.knowledgeService.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get knowledge batch for search targets: %v", err)
|
||||
return targets, nil // Return what we have, don't fail
|
||||
}
|
||||
|
||||
// Group knowledge IDs by their KB, excluding those already covered by full KB search
|
||||
kbToKnowledgeIDs := make(map[string][]string)
|
||||
for _, k := range knowledgeList {
|
||||
if k == nil || k.KnowledgeBaseID == "" {
|
||||
continue
|
||||
}
|
||||
// Skip if this KB is already fully searched
|
||||
if fullKBSet[k.KnowledgeBaseID] {
|
||||
continue
|
||||
}
|
||||
kbToKnowledgeIDs[k.KnowledgeBaseID] = append(kbToKnowledgeIDs[k.KnowledgeBaseID], k.ID)
|
||||
}
|
||||
|
||||
// Create SearchTargetTypeKnowledge targets for each KB with specific files
|
||||
for kbID, kidList := range kbToKnowledgeIDs {
|
||||
targets = append(targets, &types.SearchTarget{
|
||||
Type: types.SearchTargetTypeKnowledge,
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeIDs: kidList,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Built %d search targets: %d full KB, %d partial KB",
|
||||
len(targets), len(knowledgeBaseIDs), len(targets)-len(knowledgeBaseIDs))
|
||||
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
// KnowledgeQAByEvent processes knowledge QA through a series of events in the pipeline
|
||||
func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
|
||||
chatManage *types.ChatManage, eventList []types.EventType,
|
||||
@@ -697,8 +804,8 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
|
||||
defer span.End()
|
||||
|
||||
logger.Info(ctx, "Start processing knowledge base question answering through events")
|
||||
logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, knowledge base ID: %s, query: %s",
|
||||
chatManage.SessionID, chatManage.KnowledgeBaseID, chatManage.Query)
|
||||
logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s",
|
||||
chatManage.SessionID, chatManage.Query)
|
||||
|
||||
// Prepare method list for logging and tracing
|
||||
methods := []string{}
|
||||
@@ -769,7 +876,6 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
|
||||
chatManage := &types.ChatManage{
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
|
||||
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
|
||||
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
|
||||
@@ -800,10 +906,10 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
|
||||
|
||||
// Use specific event list, only including retrieval-related events, not LLM summarization
|
||||
searchEvents := []types.EventType{
|
||||
types.CHUNK_SEARCH, // Vector search
|
||||
types.CHUNK_RERANK, // Rerank search results
|
||||
types.CHUNK_MERGE, // Merge search results
|
||||
types.FILTER_TOP_K, // Filter top K results
|
||||
types.CHUNK_SEARCH, // Vector search
|
||||
types.CHUNK_RERANK, // Rerank search results
|
||||
types.CHUNK_MERGE, // Merge search results
|
||||
types.FILTER_TOP_K, // Filter top K results
|
||||
}
|
||||
|
||||
ctx, span := tracing.ContextWithSpan(ctx, "SessionService.SearchKnowledge")
|
||||
@@ -895,13 +1001,14 @@ func (s *sessionService) AgentQA(
|
||||
|
||||
// Create runtime AgentConfig by merging session and tenant configs
|
||||
// Tenant config provides the runtime parameters (MaxIterations, Temperature, Tools, Models)
|
||||
// Session config provides KnowledgeBases
|
||||
// Session config provides KnowledgeBases and KnowledgeIDs
|
||||
agentConfig := &types.AgentConfig{
|
||||
MaxIterations: tenantInfo.AgentConfig.MaxIterations,
|
||||
ReflectionEnabled: tenantInfo.AgentConfig.ReflectionEnabled,
|
||||
AllowedTools: tools.DefaultAllowedTools(),
|
||||
Temperature: tenantInfo.AgentConfig.Temperature,
|
||||
KnowledgeBases: session.AgentConfig.KnowledgeBases, // Use session's knowledge bases
|
||||
KnowledgeIDs: session.AgentConfig.KnowledgeIDs, // Use session's knowledge IDs (individual documents)
|
||||
WebSearchEnabled: session.AgentConfig.WebSearchEnabled, // Web search enabled from session config
|
||||
}
|
||||
|
||||
@@ -919,43 +1026,42 @@ func (s *sessionService) AgentQA(
|
||||
|
||||
logger.Infof(ctx, "Merged agent config from tenant %d and session %s", tenantInfo.ID, sessionID)
|
||||
|
||||
// Log knowledge IDs if present
|
||||
if len(agentConfig.KnowledgeIDs) > 0 {
|
||||
logger.Infof(ctx, "Agent configured with %d individual knowledge ID(s): %v",
|
||||
len(agentConfig.KnowledgeIDs), agentConfig.KnowledgeIDs)
|
||||
}
|
||||
|
||||
// Determine knowledge bases for agent
|
||||
// Priority: Session.AgentConfig.KnowledgeBases > Session.KnowledgeBaseID > All tenant knowledge bases
|
||||
if len(agentConfig.KnowledgeBases) == 0 {
|
||||
// Exception: If KnowledgeIDs are specified, don't auto-add all KBs (let buildSearchTargets handle it)
|
||||
if len(agentConfig.KnowledgeBases) == 0 && len(agentConfig.KnowledgeIDs) == 0 {
|
||||
if session.KnowledgeBaseID != "" {
|
||||
// Use session's knowledge base as fallback
|
||||
agentConfig.KnowledgeBases = []string{session.KnowledgeBaseID}
|
||||
logger.Infof(ctx, "Using session's knowledge base for agent: %s", session.KnowledgeBaseID)
|
||||
} else {
|
||||
// Default to all knowledge bases under the tenant
|
||||
logger.Infof(ctx, "No knowledge bases specified, fetching all knowledge bases for tenant")
|
||||
allKBs, err := s.knowledgeBaseService.ListKnowledgeBases(ctx)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to list knowledge bases for tenant: %v", err)
|
||||
return fmt.Errorf("failed to list knowledge bases: %w", err)
|
||||
}
|
||||
|
||||
if len(allKBs) == 0 {
|
||||
logger.Warnf(ctx, "No knowledge bases available for agent session: %s", sessionID)
|
||||
return errors.New("no knowledge bases available for agent")
|
||||
}
|
||||
|
||||
// Extract knowledge base IDs
|
||||
agentConfig.KnowledgeBases = make([]string, len(allKBs))
|
||||
for i, kb := range allKBs {
|
||||
if kb == nil {
|
||||
continue
|
||||
}
|
||||
agentConfig.KnowledgeBases[i] = kb.ID
|
||||
}
|
||||
logger.Infof(ctx, "Agent defaulting to all %d knowledge base(s) in tenant: %v",
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
// Allow running without knowledge bases (Pure Agent mode)
|
||||
logger.Infof(ctx, "No knowledge bases specified for agent, running in pure agent mode")
|
||||
}
|
||||
} else if len(agentConfig.KnowledgeIDs) > 0 && len(agentConfig.KnowledgeBases) == 0 {
|
||||
// User specified individual files but no KBs - don't auto-add all KBs
|
||||
logger.Infof(ctx, "Agent configured with %d individual knowledge ID(s), no KB auto-expansion",
|
||||
len(agentConfig.KnowledgeIDs))
|
||||
} else {
|
||||
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
}
|
||||
|
||||
// Build search targets for agent (pre-compute once to avoid repeated queries)
|
||||
searchTargets, err := s.buildSearchTargets(ctx, tenantInfo.ID, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to build search targets for agent: %v", err)
|
||||
// Continue without search targets, the tool will handle empty targets
|
||||
}
|
||||
agentConfig.SearchTargets = searchTargets
|
||||
logger.Infof(ctx, "Agent search targets built: %d targets", len(searchTargets))
|
||||
|
||||
summaryModelID := session.SummaryModelID
|
||||
if summaryModelID == "" && tenantInfo.ConversationConfig != nil {
|
||||
summaryModelID = tenantInfo.ConversationConfig.SummaryModelID
|
||||
|
||||
Reference in New Issue
Block a user