Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipline/search_entity.go
T
wizardchen 4a9977b11f feat: enhance document loading and state management in doc-content component
- Introduced a loading state (`loadingChunks`) to manage document chunk loading more effectively.
- Updated the scroll handling logic to prevent multiple requests while loading chunks.
- Adjusted the pagination logic to ensure correct page increments based on the total document count.

This update improves the user experience by ensuring smoother document loading and better state management during scrolling interactions.
2026-03-02 21:21:49 +08:00

228 lines
6.9 KiB
Go

package chatpipline
import (
"context"
"sync"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
type PluginSearchEntity struct {
graphRepo interfaces.RetrieveGraphRepository
chunkRepo interfaces.ChunkRepository
knowledgeRepo interfaces.KnowledgeRepository
}
// NewPluginSearchEntity creates a new plugin search entity
func NewPluginSearchEntity(
eventManager *EventManager,
graphRepository interfaces.RetrieveGraphRepository,
chunkRepository interfaces.ChunkRepository,
knowledgeRepository interfaces.KnowledgeRepository,
) *PluginSearchEntity {
res := &PluginSearchEntity{
graphRepo: graphRepository,
chunkRepo: chunkRepository,
knowledgeRepo: knowledgeRepository,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the list of event types this plugin responds to
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
return []types.EventType{types.ENTITY_SEARCH}
}
// OnEvent processes triggered events
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
entity := chatManage.Entity
if len(entity) == 0 {
logger.Infof(ctx, "No entity found")
return next()
}
// Use EntityKBIDs (knowledge bases with ExtractConfig enabled)
knowledgeBaseIDs := chatManage.EntityKBIDs
// Use EntityKnowledge (KnowledgeID -> KnowledgeBaseID mapping for graph-enabled files)
entityKnowledge := chatManage.EntityKnowledge
if len(knowledgeBaseIDs) == 0 && len(entityKnowledge) == 0 {
logger.Warnf(ctx, "No knowledge base IDs or knowledge IDs with ExtractConfig enabled for entity search")
return next()
}
// Parallel search across multiple knowledge bases and individual files
var wg sync.WaitGroup
var mu sync.Mutex
var allNodes []*types.GraphNode
var allRelations []*types.GraphRelation
// If specific KnowledgeIDs are provided, search by individual files
if len(entityKnowledge) > 0 {
logger.Infof(ctx, "Searching entities across %d knowledge file(s)", len(entityKnowledge))
for knowledgeID, kbID := range entityKnowledge {
wg.Add(1)
go func(knowledgeBaseID, knowledgeID string) {
defer wg.Done()
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{
KnowledgeBase: knowledgeBaseID,
Knowledge: knowledgeID,
}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search entity in Knowledge %s: %v", knowledgeID, err)
return
}
logger.Infof(
ctx,
"Knowledge %s entity search result count: %d nodes, %d relations",
knowledgeID,
len(graph.Node),
len(graph.Relation),
)
mu.Lock()
allNodes = append(allNodes, graph.Node...)
allRelations = append(allRelations, graph.Relation...)
mu.Unlock()
}(kbID, knowledgeID)
}
} else {
// Otherwise, search by knowledge base
logger.Infof(ctx, "Searching entities across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
for _, kbID := range knowledgeBaseIDs {
wg.Add(1)
go func(knowledgeBaseID string) {
defer wg.Done()
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: knowledgeBaseID}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search entity in KB %s: %v", knowledgeBaseID, err)
return
}
logger.Infof(
ctx,
"KB %s entity search result count: %d nodes, %d relations",
knowledgeBaseID,
len(graph.Node),
len(graph.Relation),
)
mu.Lock()
allNodes = append(allNodes, graph.Node...)
allRelations = append(allRelations, graph.Relation...)
mu.Unlock()
}(kbID)
}
}
wg.Wait()
// Merge graph data
chatManage.GraphResult = &types.GraphData{
Node: allNodes,
Relation: allRelations,
}
logger.Infof(ctx, "Total entity search result: %d nodes, %d relations", len(allNodes), len(allRelations))
chunkIDs := filterSeenChunk(ctx, chatManage.GraphResult, chatManage.SearchResult)
if len(chunkIDs) == 0 {
logger.Infof(ctx, "No new chunk found")
return next()
}
chunks, err := p.chunkRepo.ListChunksByID(ctx, types.MustTenantIDFromContext(ctx), chunkIDs)
if err != nil {
logger.Errorf(ctx, "Failed to list chunks, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
knowledgeIDs := []string{}
for _, chunk := range chunks {
knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID)
}
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(
ctx,
types.MustTenantIDFromContext(ctx),
knowledgeIDs,
)
if err != nil {
logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
knowledgeMap := map[string]*types.Knowledge{}
for _, knowledge := range knowledges {
knowledgeMap[knowledge.ID] = knowledge
}
for _, chunk := range chunks {
searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID])
chatManage.SearchResult = append(chatManage.SearchResult, searchResult)
}
// remove duplicate results
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
if len(chatManage.SearchResult) == 0 {
logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID)
return ErrSearchNothing
}
logger.Infof(
ctx,
"search entity result count: %d, session_id: %s",
len(chatManage.SearchResult),
chatManage.SessionID,
)
return next()
}
// filterSeenChunk filters seen chunks from the graph
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
seen := map[string]bool{}
for _, chunk := range searchResult {
seen[chunk.ID] = true
}
logger.Infof(ctx, "filterSeenChunk: seen count: %d", len(seen))
chunkIDs := []string{}
for _, node := range graph.Node {
for _, chunkID := range node.Chunks {
if seen[chunkID] {
continue
}
seen[chunkID] = true
chunkIDs = append(chunkIDs, chunkID)
}
}
logger.Infof(ctx, "filterSeenChunk: new chunkIDs count: %d", len(chunkIDs))
return chunkIDs
}
// chunk2SearchResult converts a chunk to a search result
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
return &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
KnowledgeTitle: knowledge.Title,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
Seq: chunk.ChunkIndex,
Score: 1.0,
MatchType: types.MatchTypeGraph,
Metadata: knowledge.GetMetadata(),
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
KnowledgeFilename: knowledge.FileName,
KnowledgeSource: knowledge.Source,
ChunkMetadata: chunk.Metadata,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
}
}