feat: 新增异步任务提取服务

This commit is contained in:
begoniezhao
2025-09-24 12:15:17 +08:00
parent 31d19095f4
commit b6d8be634a
30 changed files with 2514 additions and 117 deletions
@@ -0,0 +1,231 @@
package neo4j
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
)
type Neo4jRepository struct {
driver neo4j.Driver
nodePrefix string
}
func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository {
return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"}
}
func _remove_hyphen(s string) string {
return strings.ReplaceAll(s, "-", "_")
}
func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
res := make([]string, 0)
for _, label := range namespace.Labels() {
res = append(res, n.nodePrefix+_remove_hyphen(label))
}
return res
}
func (n *Neo4jRepository) Label(namespace types.NameSpace) string {
labels := n.Labels(namespace)
return strings.Join(labels, ":")
}
// AddGraph implements interfaces.RetrieveGraphRepository.
func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error {
if n.driver == nil {
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
return nil
}
for _, graph := range graphs {
if err := n.addGraph(ctx, namespace, graph); err != nil {
return err
}
}
return nil
}
func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error {
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close(ctx)
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
node_import_query := `
UNWIND $data AS row
CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node
SET node.chunks = apoc.coll.union(node.chunks, row.chunks)
RETURN distinct 'done' AS result
`
nodeData := []map[string]interface{}{}
for _, node := range graph.Node {
nodeData = append(nodeData, map[string]interface{}{
"name": node.Name,
"knowledge_id": namespace.Knowledge,
"props": map[string][]string{"attributes": node.Attributes},
"chunks": node.Chunks,
"labels": n.Labels(namespace),
})
}
if _, err := tx.Run(ctx, node_import_query, map[string]interface{}{"data": nodeData}); err != nil {
return nil, fmt.Errorf("failed to create nodes: %v", err)
}
rel_import_query := `
UNWIND $data AS row
CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source
CALL apoc.merge.node(row.target_labels, {name: row.target, kg: row.knowledge_id}, {}, {}) YIELD node as target
CALL apoc.merge.relationship(source, row.type, {}, row.attributes, target) YIELD rel
RETURN distinct 'done'
`
relData := []map[string]interface{}{}
for _, rel := range graph.Relation {
relData = append(relData, map[string]interface{}{
"source": rel.Node1,
"target": rel.Node2,
"knowledge_id": namespace.Knowledge,
"type": rel.Type,
"source_labels": n.Labels(namespace),
"target_labels": n.Labels(namespace),
})
}
if _, err := tx.Run(ctx, rel_import_query, map[string]interface{}{"data": relData}); err != nil {
return nil, fmt.Errorf("failed to create relationships: %v", err)
}
return nil, nil
})
if err != nil {
logger.Errorf(ctx, "failed to add graph: %v", err)
return err
}
return nil
}
// DelGraph implements interfaces.RetrieveGraphRepository.
func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error {
if n.driver == nil {
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
return nil
}
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close(ctx)
result, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
for _, namespace := range namespaces {
labelExpr := n.Label(namespace)
deleteRelsQuery := `
CALL apoc.periodic.iterate(
"MATCH (n:` + labelExpr + ` {kg: $knowledge_id})-[r]-(m:` + labelExpr + ` {kg: $knowledge_id}) RETURN r",
"DELETE r",
{batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}}
) YIELD batches, total
RETURN total
`
if _, err := tx.Run(ctx, deleteRelsQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil {
return nil, fmt.Errorf("failed to delete relationships: %v", err)
}
deleteNodesQuery := `
CALL apoc.periodic.iterate(
"MATCH (n:` + labelExpr + ` {kg: $knowledge_id}) RETURN n",
"DELETE n",
{batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}}
) YIELD batches, total
RETURN total
`
if _, err := tx.Run(ctx, deleteNodesQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil {
return nil, fmt.Errorf("failed to delete nodes: %v", err)
}
}
return nil, nil
})
if err != nil {
return err
}
logger.Infof(ctx, "delete graph result: %v", result)
return nil
}
func (n *Neo4jRepository) SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) {
if n.driver == nil {
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
return nil, nil
}
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
defer session.Close(ctx)
result, err := session.ExecuteRead(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
labelExpr := n.Label(namespace)
query := `
MATCH (n:` + labelExpr + `)-[r]-(m:` + labelExpr + `)
WHERE ANY(nodeText IN $nodes WHERE n.name CONTAINS nodeText)
RETURN n, r, m
`
params := map[string]interface{}{"nodes": nodes}
result, err := tx.Run(ctx, query, params)
if err != nil {
return nil, fmt.Errorf("failed to run query: %v", err)
}
graphData := &types.GraphData{}
nodeSeen := make(map[string]bool)
for result.Next(ctx) {
record := result.Record()
node, _ := record.Get("n")
rel, _ := record.Get("r")
targetNode, _ := record.Get("m")
nodeData := node.(neo4j.Node)
targetNodeData := targetNode.(neo4j.Node)
// Convert node to types.Node
for _, n := range []neo4j.Node{nodeData, targetNodeData} {
nameStr := n.Props["name"].(string)
if _, ok := nodeSeen[nameStr]; !ok {
nodeSeen[nameStr] = true
graphData.Node = append(graphData.Node, &types.GraphNode{
Name: nameStr,
Chunks: listI2listS(n.Props["chunks"].([]interface{})),
Attributes: listI2listS(n.Props["attributes"].([]interface{})),
})
}
}
// Convert relationship to types.Relation
relData := rel.(neo4j.Relationship)
graphData.Relation = append(graphData.Relation, &types.GraphRelation{
Node1: nodeData.Props["name"].(string),
Node2: targetNodeData.Props["name"].(string),
Type: relData.Type,
})
}
return graphData, nil
})
if err != nil {
logger.Errorf(ctx, "search node failed: %v", err)
return nil, err
}
return result.(*types.GraphData), nil
}
func listI2listS(list []any) []string {
result := make([]string, len(list))
for i, v := range list {
result[i] = fmt.Sprintf("%v", v)
}
return result
}
func mapI2mapS(prop map[string]any) map[string]string {
attributes := make(map[string]string)
for k, v := range prop {
attributes[k] = fmt.Sprintf("%v", v)
}
return attributes
}
@@ -0,0 +1,499 @@
package chatpipline
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"strings"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginExtractEntity is a plugin for extracting entities from user queries
// It uses historical dialog context and large language models to identify key entities in the user's original query
type PluginExtractEntity struct {
modelService interfaces.ModelService // Model service for calling large language models
template *types.PromptTemplateStructured // Template for generating prompts
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
}
// NewPluginRewrite creates a new query rewriting plugin instance
// Also registers the plugin with the event manager
func NewPluginExtractEntity(
eventManager *EventManager,
modelService interfaces.ModelService,
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
config *config.Config,
) *PluginExtractEntity {
res := &PluginExtractEntity{
modelService: modelService,
template: config.ExtractManager.ExtractEntity,
knowledgeBaseRepo: knowledgeBaseRepo,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the list of event types this plugin responds to
// This plugin only responds to REWRITE_QUERY events
func (p *PluginExtractEntity) ActivationEvents() []types.EventType {
return []types.EventType{types.REWRITE_QUERY}
}
// OnEvent processes triggered events
// When receiving a REWRITE_QUERY event, it rewrites the user query using conversation history and the language model
func (p *PluginExtractEntity) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
logger.Debugf(ctx, "skipping extract entity, neo4j is disabled")
return next()
}
query := chatManage.Query
model, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
kb, err := p.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chatManage.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
return next()
}
if kb.ExtractConfig == nil {
logger.Warnf(ctx, "failed to get extract config")
return next()
}
template := &types.PromptTemplateStructured{
Description: p.template.Description,
Examples: p.template.Examples,
}
extractor := NewExtractor(model, template)
graph, err := extractor.Extract(ctx, query)
if err != nil {
logger.Errorf(ctx, "Failed to extract entities, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
nodes := []string{}
for _, node := range graph.Node {
nodes = append(nodes, node.Name)
}
logger.Debugf(ctx, "extracted node: %v", nodes)
chatManage.Entity = nodes
return next()
}
type Extractor struct {
chat chat.Chat
formater *Formater
template *types.PromptTemplateStructured
chatOpt *chat.ChatOptions
}
func NewExtractor(
chatModel chat.Chat,
template *types.PromptTemplateStructured,
) Extractor {
think := false
return Extractor{
chat: chatModel,
formater: NewFormater(),
template: template,
chatOpt: &chat.ChatOptions{
Temperature: 0.3,
MaxTokens: 4096,
Thinking: &think,
},
}
}
func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) {
generator := NewQAPromptGenerator(e.formater, e.template)
// logger.Debugf(ctx, "chat system: %s", generator.System(ctx))
// logger.Debugf(ctx, "chat user: %s", generator.User(ctx, content))
chatResponse, err := e.chat.Chat(ctx, generator.Render(ctx, content), e.chatOpt)
if err != nil {
logger.Errorf(ctx, "failed to chat: %v", err)
return nil, err
}
graph, err := e.formater.ParseGraph(ctx, chatResponse.Content)
if err != nil {
logger.Errorf(ctx, "failed to parse graph: %v", err)
return nil, err
}
// e.RemoveUnknownRelation(ctx, graph)
return graph, nil
}
func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) {
relationType := make(map[string]bool)
for _, tag := range e.template.Tags {
relationType[tag] = true
}
relationNew := make([]*types.GraphRelation, 0)
for _, relation := range graph.Relation {
if _, ok := relationType[relation.Type]; ok {
relationNew = append(relationNew, relation)
} else {
logger.Infof(ctx, "Unknown relation type %s with %v, ignore it", relation.Type, e.template.Tags)
}
}
graph.Relation = relationNew
}
type QAPromptGenerator struct {
Formater *Formater
Template *types.PromptTemplateStructured
ExamplesHeading string
QuestionHeading string
QuestionPrefix string
AnswerPrefix string
}
func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator {
return &QAPromptGenerator{
Formater: formater,
Template: template,
ExamplesHeading: "# Examples",
QuestionHeading: "# Question",
QuestionPrefix: "Q: ",
AnswerPrefix: "A: ",
}
}
func (qa *QAPromptGenerator) System(ctx context.Context) string {
promptLines := []string{}
if len(qa.Template.Tags) == 0 {
promptLines = append(promptLines, qa.Template.Description)
} else {
tags, _ := json.Marshal(qa.Template.Tags)
promptLines = append(promptLines, fmt.Sprintf(qa.Template.Description, string(tags)))
}
if len(qa.Template.Examples) > 0 {
promptLines = append(promptLines, qa.ExamplesHeading)
for _, example := range qa.Template.Examples {
// Question
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, strings.TrimSpace(example.Text)))
// Answer
answer, err := qa.Formater.formatExtraction(example.Node, example.Relation)
if err != nil {
return ""
}
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.AnswerPrefix, answer))
// new line
promptLines = append(promptLines, "")
}
}
return strings.Join(promptLines, "\n")
}
func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
promptLines := []string{}
promptLines = append(promptLines, qa.QuestionHeading)
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, question))
promptLines = append(promptLines, qa.AnswerPrefix)
return strings.Join(promptLines, "\n")
}
func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message {
return []chat.Message{
{
Role: "system",
Content: qa.System(ctx),
},
{
Role: "user",
Content: qa.User(ctx, question),
},
}
}
type FormatType string
const (
FormatTypeJSON FormatType = "json"
FormatTypeYAML FormatType = "yaml"
)
const (
_FENCE_START = "```"
_LANGUAGE_TAG = `(?P<lang>[A-Za-z0-9_+-]+)?`
_FENCE_NEWLINE = `(?:\s*\n)?`
_FENCE_BODY = `(?P<body>[\s\S]*?)`
_FENCE_END = "```"
)
var _FENCE_RE = regexp.MustCompile(
_FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END,
)
type Formater struct {
attributeSuffix string
formatType FormatType
useFences bool
nodePrefix string
relationSource string
relationTarget string
relationPrefix string
}
func NewFormater() *Formater {
return &Formater{
attributeSuffix: "_attributes",
formatType: FormatTypeJSON,
useFences: true,
nodePrefix: "entity",
relationSource: "entity1",
relationTarget: "entity2",
relationPrefix: "relation",
}
}
func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) {
items := make([]map[string]interface{}, 0)
for _, node := range nodes {
item := map[string]interface{}{
f.nodePrefix: node.Name,
}
if len(node.Attributes) > 0 {
item[fmt.Sprintf("%s%s", f.nodePrefix, f.attributeSuffix)] = node.Attributes
}
items = append(items, item)
}
for _, relation := range relations {
item := map[string]interface{}{
f.relationSource: relation.Node1,
f.relationTarget: relation.Node2,
f.relationPrefix: relation.Type,
}
items = append(items, item)
}
formatted := ""
switch f.formatType {
default:
formattedBytes, err := json.MarshalIndent(items, "", " ")
if err != nil {
return "", err
}
formatted = string(formattedBytes)
}
if f.useFences {
formatted = f.addFences(formatted)
}
return formatted, nil
}
func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) {
if text == "" {
return nil, errors.New("Empty or invalid input string.")
}
content := f.extractContent(ctx, text)
// logger.Debugf(ctx, "Extracted content: %s", content)
if content == "" {
return nil, errors.New("Empty or invalid input string.")
}
var parsed interface{}
var err error
if f.formatType == FormatTypeJSON {
err = json.Unmarshal([]byte(content), &parsed)
}
if err != nil {
return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
}
if parsed == nil {
return nil, fmt.Errorf("Content must be a list of extractions or a dict.")
}
var items []interface{}
if parsedMap, ok := parsed.(map[string]interface{}); ok {
items = []interface{}{parsedMap}
} else if parsedList, ok := parsed.([]interface{}); ok {
items = parsedList
} else {
return nil, fmt.Errorf("Expected list or dict, got %T", parsed)
}
itemsList := make([]map[string]interface{}, 0)
for _, item := range items {
if itemMap, ok := item.(map[string]interface{}); ok {
itemsList = append(itemsList, itemMap)
} else {
return nil, fmt.Errorf("Each item in the sequence must be a mapping.")
}
}
return itemsList, nil
}
func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphData, error) {
matchData, err := f.parseOutput(ctx, text)
if err != nil {
return nil, err
}
if len(matchData) == 0 {
logger.Debugf(ctx, "Received empty extraction data.")
return &types.GraphData{}, nil
}
// mm, _ := json.Marshal(matchData)
// logger.Debugf(ctx, "Parsed graph data: %s", string(mm))
var nodes []*types.GraphNode
var relations []*types.GraphRelation
for _, group := range matchData {
switch {
case group[f.nodePrefix] != nil:
attributes := make([]string, 0)
attributesKey := f.nodePrefix + f.attributeSuffix
if attr, ok := group[attributesKey].([]interface{}); ok {
for _, v := range attr {
attributes = append(attributes, fmt.Sprintf("%v", v))
}
}
nodes = append(nodes, &types.GraphNode{
Name: fmt.Sprintf("%v", group[f.nodePrefix]),
Attributes: attributes,
})
case group[f.relationSource] != nil && group[f.relationTarget] != nil:
relations = append(relations, &types.GraphRelation{
Node1: fmt.Sprintf("%v", group[f.relationSource]),
Node2: fmt.Sprintf("%v", group[f.relationTarget]),
Type: fmt.Sprintf("%v", group[f.relationPrefix]),
})
default:
logger.Warnf(ctx, "Unsupported graph group: %v", group)
continue
}
}
graph := &types.GraphData{
Node: nodes,
Relation: relations,
}
f.rebuildGraph(ctx, graph)
return graph, nil
}
func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) {
nodeMap := make(map[string]*types.GraphNode)
nodes := make([]*types.GraphNode, 0, len(graph.Node))
for _, node := range graph.Node {
if prenode, ok := nodeMap[node.Name]; ok {
logger.Infof(ctx, "Duplicate node ID: %s, merge attribute", node.Name)
// 修复panic:检查Attributes是否为nil
if node.Attributes == nil {
node.Attributes = make([]string, 0)
}
if prenode.Attributes != nil {
for _, attr := range prenode.Attributes {
node.Attributes = append(node.Attributes, attr)
}
}
continue
}
nodeMap[node.Name] = node
nodes = append(nodes, node)
}
relations := make([]*types.GraphRelation, 0, len(graph.Relation))
for _, relation := range graph.Relation {
if relation.Node1 == relation.Node2 {
logger.Infof(ctx, "Duplicate relation, ignore it")
continue
}
if _, ok := nodeMap[relation.Node1]; !ok {
node := &types.GraphNode{Name: relation.Node1}
nodes = append(nodes, node)
nodeMap[relation.Node1] = node
logger.Infof(ctx, "Add unknown source node ID: %s", relation.Node1)
}
if _, ok := nodeMap[relation.Node2]; !ok {
node := &types.GraphNode{Name: relation.Node2}
nodes = append(nodes, node)
nodeMap[relation.Node2] = node
logger.Infof(ctx, "Add unknown target node ID: %s", relation.Node2)
}
relations = append(relations, relation)
}
*graph = types.GraphData{
Node: nodes,
Relation: relations,
}
}
func (f *Formater) extractContent(ctx context.Context, text string) string {
if !f.useFences {
return strings.TrimSpace(text)
}
validTags := map[FormatType]map[string]struct{}{
FormatTypeYAML: {"yaml": {}, "yml": {}},
FormatTypeJSON: {"json": {}},
}
matches := _FENCE_RE.FindAllStringSubmatch(text, -1)
var candidates []string
for _, match := range matches {
lang := match[1]
body := match[2]
if f.isValidLanguageTag(lang, validTags) {
candidates = append(candidates, body)
}
}
switch {
case len(candidates) == 1:
return strings.TrimSpace(candidates[0])
case len(candidates) > 1:
logger.Warnf(ctx, "multiple candidates found: %d", len(candidates))
return strings.TrimSpace(candidates[0])
case len(matches) == 1:
logger.Debugf(ctx, "no candidate found, use first match without language tag: %s", matches[0][1])
return strings.TrimSpace(matches[0][2])
case len(matches) > 1:
logger.Warnf(ctx, "multiple matches found: %d", len(matches))
return strings.TrimSpace(matches[0][2])
default:
logger.Warnf(ctx, "no match found")
return strings.TrimSpace(text)
}
}
func (f *Formater) addFences(content string) string {
content = strings.TrimSpace(content)
return fmt.Sprintf("```%s\n%s\n```", f.formatType, content)
}
func (f *Formater) isValidLanguageTag(lang string, validTags map[FormatType]map[string]struct{}) bool {
if lang == "" {
return true
}
tag := strings.TrimSpace(strings.ToLower(lang))
validSet, ok := validTags[f.formatType]
if !ok {
return false
}
_, exists := validSet[tag]
return exists
}
@@ -77,6 +77,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
}
chatManage.SearchResult = append(chatManage.SearchResult, searchResults...)
}
// remove duplicate results
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
@@ -0,0 +1,136 @@
package chatpipline
import (
"context"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
type PluginSearchEntity struct {
graphRepo interfaces.RetrieveGraphRepository
chunkRepo interfaces.ChunkRepository
knowledgeRepo interfaces.KnowledgeRepository
}
func NewPluginSearchEntity(
eventManager *EventManager,
graphRepository interfaces.RetrieveGraphRepository,
chunkRepository interfaces.ChunkRepository,
knowledgeRepository interfaces.KnowledgeRepository,
) *PluginSearchEntity {
res := &PluginSearchEntity{
graphRepo: graphRepository,
chunkRepo: chunkRepository,
knowledgeRepo: knowledgeRepository,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the event types this plugin handles
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
return []types.EventType{types.ENTITY_SEARCH}
}
// OnEvent handles search events in the chat pipeline
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
entity := chatManage.Entity
if len(entity) == 0 {
logger.Infof(ctx, "No entity found")
return next()
}
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: chatManage.KnowledgeBaseID}, entity)
if err != nil {
logger.Errorf(ctx, "Failed to search node, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
chatManage.GraphResult = graph
logger.Infof(ctx, "search entity result count: %d", len(graph.Node))
// graphStr, _ := json.Marshal(graph)
// logger.Debugf(ctx, "search entity result: %s", string(graphStr))
chunkIDs := filterSeenChunk(ctx, graph, chatManage.SearchResult)
if len(chunkIDs) == 0 {
logger.Infof(ctx, "No new chunk found")
return next()
}
chunks, err := p.chunkRepo.ListChunksByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), chunkIDs)
if err != nil {
logger.Errorf(ctx, "Failed to list chunks, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
knowledgeIDs := []string{}
for _, chunk := range chunks {
knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID)
}
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledgeIDs)
if err != nil {
logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err)
return next()
}
knowledgeMap := map[string]*types.Knowledge{}
for _, knowledge := range knowledges {
knowledgeMap[knowledge.ID] = knowledge
}
for _, chunk := range chunks {
searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID])
chatManage.SearchResult = append(chatManage.SearchResult, searchResult)
}
// remove duplicate results
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
if len(chatManage.SearchResult) == 0 {
logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID)
return ErrSearchNothing
}
logger.Infof(ctx, "search entity result count: %d, session_id: %s", len(chatManage.SearchResult), chatManage.SessionID)
return next()
}
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
seen := map[string]bool{}
for _, chunk := range searchResult {
seen[chunk.ID] = true
}
logger.Infof(ctx, "filterSeenChunk: seen count: %d", len(seen))
chunkIDs := []string{}
for _, node := range graph.Node {
for _, chunkID := range node.Chunks {
if seen[chunkID] {
continue
}
seen[chunkID] = true
chunkIDs = append(chunkIDs, chunkID)
}
}
logger.Infof(ctx, "filterSeenChunk: new chunkIDs count: %d", len(chunkIDs))
return chunkIDs
}
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
return &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
KnowledgeTitle: knowledge.Title,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
Seq: chunk.ChunkIndex,
Score: 1.0,
MatchType: types.MatchTypeGraph,
Metadata: knowledge.GetMetadata(),
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
KnowledgeFilename: knowledge.FileName,
KnowledgeSource: knowledge.Source,
}
}
+137
View File
@@ -0,0 +1,137 @@
package service
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
func NewChunkExtractTask(ctx context.Context, client *asynq.Client, tenantID uint, chunkID string, modelID string) error {
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
logger.Debugf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
return nil
}
payload, err := json.Marshal(types.ExtractChunkPayload{
TenantID: tenantID,
ChunkID: chunkID,
ModelID: modelID,
})
if err != nil {
return err
}
task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.MaxRetry(3))
info, err := client.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "failed to enqueue task: %v", err)
return fmt.Errorf("failed to enqueue task: %v", err)
}
logger.Infof(ctx, "enqueued task: id=%s queue=%s chunk=%s", info.ID, info.Queue, chunkID)
return nil
}
type ChunkExtractService struct {
template *types.PromptTemplateStructured
modelService interfaces.ModelService
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
chunkRepo interfaces.ChunkRepository
graphEngine interfaces.RetrieveGraphRepository
}
func NewChunkExtractService(
config *config.Config,
modelService interfaces.ModelService,
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
chunkRepo interfaces.ChunkRepository,
graphEngine interfaces.RetrieveGraphRepository,
) interfaces.Extracter {
generator := chatpipline.NewQAPromptGenerator(chatpipline.NewFormater(), config.ExtractManager.ExtractGraph)
ctx := context.Background()
logger.Debugf(ctx, "chunk extract system prompt: %s", generator.System(ctx))
logger.Debugf(ctx, "chunk extract user prompt: %s", generator.User(ctx, "demo"))
return &ChunkExtractService{
template: config.ExtractManager.ExtractGraph,
modelService: modelService,
knowledgeBaseRepo: knowledgeBaseRepo,
chunkRepo: chunkRepo,
graphEngine: graphEngine,
}
}
func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error {
var p types.ExtractChunkPayload
if err := json.Unmarshal(t.Payload(), &p); err != nil {
logger.Errorf(ctx, "failed to unmarshal task payload: %v", err)
return err
}
ctx = logger.WithRequestID(ctx, uuid.New().String())
ctx = logger.WithField(ctx, "extract", p.ChunkID)
ctx = context.WithValue(ctx, types.TenantIDContextKey, p.TenantID)
chunk, err := s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
if err != nil {
logger.Errorf(ctx, "failed to get chunk: %v", err)
return err
}
kb, err := s.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chunk.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
return err
}
if kb.ExtractConfig == nil {
logger.Warnf(ctx, "failed to get extract config")
return err
}
chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID)
if err != nil {
logger.Errorf(ctx, "failed to get chat model: %v", err)
return err
}
template := &types.PromptTemplateStructured{
Description: s.template.Description,
Tags: kb.ExtractConfig.Tags,
Examples: []types.GraphData{
{
Text: kb.ExtractConfig.Text,
Node: kb.ExtractConfig.Nodes,
Relation: kb.ExtractConfig.Relations,
},
},
}
extractor := chatpipline.NewExtractor(chatModel, template)
graph, err := extractor.Extract(ctx, chunk.Content)
if err != nil {
return err
}
chunk, err = s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
if err != nil {
logger.Warnf(ctx, "graph ignore chunk %s: %v", p.ChunkID, err)
return nil
}
for _, node := range graph.Node {
node.Chunks = []string{chunk.ID}
}
if err = s.graphEngine.AddGraph(ctx,
types.NameSpace{KnowledgeBase: chunk.KnowledgeBaseID, Knowledge: chunk.KnowledgeID},
[]*types.GraphData{graph},
); err != nil {
logger.Errorf(ctx, "failed to add graph: %v", err)
return err
}
// gg, _ := json.Marshal(graph)
// logger.Infof(ctx, "extracted graph: %s", string(gg))
return nil
}
+39
View File
@@ -29,6 +29,7 @@ import (
"github.com/Tencent/WeKnora/services/docreader/src/client"
"github.com/Tencent/WeKnora/services/docreader/src/proto"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"go.opentelemetry.io/otel/attribute"
"golang.org/x/sync/errgroup"
)
@@ -61,6 +62,8 @@ type knowledgeService struct {
chunkRepo interfaces.ChunkRepository
fileSvc interfaces.FileService
modelService interfaces.ModelService
task *asynq.Client
graphEngine interfaces.RetrieveGraphRepository
}
// NewKnowledgeService creates a new knowledge service instance
@@ -74,6 +77,8 @@ func NewKnowledgeService(
chunkRepo interfaces.ChunkRepository,
fileSvc interfaces.FileService,
modelService interfaces.ModelService,
task *asynq.Client,
graphEngine interfaces.RetrieveGraphRepository,
) (interfaces.KnowledgeService, error) {
return &knowledgeService{
config: config,
@@ -85,6 +90,8 @@ func NewKnowledgeService(
chunkRepo: chunkRepo,
fileSvc: fileSvc,
modelService: modelService,
task: task,
graphEngine: graphEngine,
}, nil
}
@@ -488,6 +495,16 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
return nil
})
// Delete the knowledge graph
wg.Go(func() error {
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
return err
}
return nil
})
if err = wg.Wait(); err != nil {
return err
}
@@ -561,6 +578,19 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
return nil
})
// Delete the knowledge graph
wg.Go(func() error {
namespaces := []types.NameSpace{}
for _, knowledge := range knowledgeList {
namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID})
}
if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
return err
}
return nil
})
if err = wg.Wait(); err != nil {
return err
}
@@ -1158,6 +1188,15 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
logger.Infof(ctx, "processChunks create relationship rag task")
for _, chunk := range textChunks {
err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed")
span.RecordError(err)
}
}
// Update knowledge status to completed
knowledge.ParseStatus = "completed"
knowledge.EnableStatus = "enabled"
@@ -2,12 +2,11 @@ package service
import (
"context"
"encoding/json"
"errors"
"slices"
"time"
"encoding/json"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
@@ -376,8 +375,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
// processSearchResults handles the processing of search results, optimizing database queries
func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
chunks []*types.IndexWithScore) ([]*types.SearchResult, error) {
chunks []*types.IndexWithScore,
) ([]*types.SearchResult, error) {
if len(chunks) == 0 {
return nil, nil
}
@@ -527,8 +526,8 @@ func (s *knowledgeBaseService) collectRelatedChunkIDs(chunk *types.Chunk, proces
func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk,
knowledge *types.Knowledge,
score float64,
matchType types.MatchType) *types.SearchResult {
matchType types.MatchType,
) *types.SearchResult {
return &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,