266b9a5943
- Updated agentChat API to accept optional knowledge_base_ids and agent_enabled parameters. - Introduced getSession API to retrieve session details. - Modified useStream to validate knowledge_base_ids for agent-chat requests and prepare POST body accordingly. - Enhanced createKnowledgeBase API to include additional configuration options. - Refactored chat routing to remove knowledge base ID from the chat path, allowing for dynamic knowledge base selection. - Improved session management by preventing duplicate session entries in the menu store. - Updated UI components for better session handling and display. - Added context management for agent conversations to improve state persistence across sessions.
177 lines
5.6 KiB
Go
177 lines
5.6 KiB
Go
package chatpipline
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// PluginSearch implements search functionality for chat pipeline
|
|
type PluginSearchEntity struct {
|
|
graphRepo interfaces.RetrieveGraphRepository
|
|
chunkRepo interfaces.ChunkRepository
|
|
knowledgeRepo interfaces.KnowledgeRepository
|
|
}
|
|
|
|
func NewPluginSearchEntity(
|
|
eventManager *EventManager,
|
|
graphRepository interfaces.RetrieveGraphRepository,
|
|
chunkRepository interfaces.ChunkRepository,
|
|
knowledgeRepository interfaces.KnowledgeRepository,
|
|
) *PluginSearchEntity {
|
|
res := &PluginSearchEntity{
|
|
graphRepo: graphRepository,
|
|
chunkRepo: chunkRepository,
|
|
knowledgeRepo: knowledgeRepository,
|
|
}
|
|
eventManager.Register(res)
|
|
return res
|
|
}
|
|
|
|
// ActivationEvents returns the event types this plugin handles
|
|
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
|
|
return []types.EventType{types.ENTITY_SEARCH}
|
|
}
|
|
|
|
// OnEvent handles search events in the chat pipeline
|
|
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
|
) *PluginError {
|
|
entity := chatManage.Entity
|
|
if len(entity) == 0 {
|
|
logger.Infof(ctx, "No entity found")
|
|
return next()
|
|
}
|
|
|
|
// Get knowledge base IDs list
|
|
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
|
|
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
|
|
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
|
|
logger.Infof(ctx, "No KnowledgeBaseIDs provided, falling back to single KB: %s", chatManage.KnowledgeBaseID)
|
|
}
|
|
|
|
if len(knowledgeBaseIDs) == 0 {
|
|
logger.Warnf(ctx, "No knowledge base IDs available for entity search")
|
|
return next()
|
|
}
|
|
|
|
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
|
|
|
|
// Parallel search across multiple knowledge bases
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var allNodes []*types.GraphNode
|
|
var allRelations []*types.GraphRelation
|
|
|
|
for _, kbID := range knowledgeBaseIDs {
|
|
wg.Add(1)
|
|
go func(knowledgeBaseID string) {
|
|
defer wg.Done()
|
|
|
|
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
|
|
return
|
|
}
|
|
|
|
logger.Infof(ctx, "KB %s entity search result count: %d nodes, %d relations", knowledgeBaseID, len(graph.Node), len(graph.Relation))
|
|
|
|
mu.Lock()
|
|
allNodes = append(allNodes, graph.Node...)
|
|
allRelations = append(allRelations, graph.Relation...)
|
|
mu.Unlock()
|
|
}(kbID)
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
// Merge graph data
|
|
chatManage.GraphResult = &types.GraphData{
|
|
Node: allNodes,
|
|
Relation: allRelations,
|
|
}
|
|
logger.Infof(ctx, "Total entity search result: %d nodes, %d relations", len(allNodes), len(allRelations))
|
|
|
|
chunkIDs := filterSeenChunk(ctx, chatManage.GraphResult, chatManage.SearchResult)
|
|
if len(chunkIDs) == 0 {
|
|
logger.Infof(ctx, "No new chunk found")
|
|
return next()
|
|
}
|
|
chunks, err := p.chunkRepo.ListChunksByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), chunkIDs)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to list chunks, session_id: %s, error: %v", chatManage.SessionID, err)
|
|
return next()
|
|
}
|
|
knowledgeIDs := []string{}
|
|
for _, chunk := range chunks {
|
|
knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID)
|
|
}
|
|
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledgeIDs)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err)
|
|
return next()
|
|
}
|
|
|
|
knowledgeMap := map[string]*types.Knowledge{}
|
|
for _, knowledge := range knowledges {
|
|
knowledgeMap[knowledge.ID] = knowledge
|
|
}
|
|
for _, chunk := range chunks {
|
|
searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID])
|
|
chatManage.SearchResult = append(chatManage.SearchResult, searchResult)
|
|
}
|
|
// remove duplicate results
|
|
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
|
if len(chatManage.SearchResult) == 0 {
|
|
logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID)
|
|
return ErrSearchNothing
|
|
}
|
|
logger.Infof(ctx, "search entity result count: %d, session_id: %s", len(chatManage.SearchResult), chatManage.SessionID)
|
|
return next()
|
|
}
|
|
|
|
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
|
|
seen := map[string]bool{}
|
|
for _, chunk := range searchResult {
|
|
seen[chunk.ID] = true
|
|
}
|
|
logger.Infof(ctx, "filterSeenChunk: seen count: %d", len(seen))
|
|
|
|
chunkIDs := []string{}
|
|
for _, node := range graph.Node {
|
|
for _, chunkID := range node.Chunks {
|
|
if seen[chunkID] {
|
|
continue
|
|
}
|
|
seen[chunkID] = true
|
|
chunkIDs = append(chunkIDs, chunkID)
|
|
}
|
|
}
|
|
logger.Infof(ctx, "filterSeenChunk: new chunkIDs count: %d", len(chunkIDs))
|
|
return chunkIDs
|
|
}
|
|
|
|
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
|
|
return &types.SearchResult{
|
|
ID: chunk.ID,
|
|
Content: chunk.Content,
|
|
KnowledgeID: chunk.KnowledgeID,
|
|
ChunkIndex: chunk.ChunkIndex,
|
|
KnowledgeTitle: knowledge.Title,
|
|
StartAt: chunk.StartAt,
|
|
EndAt: chunk.EndAt,
|
|
Seq: chunk.ChunkIndex,
|
|
Score: 1.0,
|
|
MatchType: types.MatchTypeGraph,
|
|
Metadata: knowledge.GetMetadata(),
|
|
ChunkType: string(chunk.ChunkType),
|
|
ParentChunkID: chunk.ParentChunkID,
|
|
ImageInfo: chunk.ImageInfo,
|
|
KnowledgeFilename: knowledge.FileName,
|
|
KnowledgeSource: knowledge.Source,
|
|
}
|
|
}
|