feat: 新增异步任务提取服务
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
)
|
||||
|
||||
type Neo4jRepository struct {
|
||||
driver neo4j.Driver
|
||||
nodePrefix string
|
||||
}
|
||||
|
||||
func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository {
|
||||
return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"}
|
||||
}
|
||||
|
||||
func _remove_hyphen(s string) string {
|
||||
return strings.ReplaceAll(s, "-", "_")
|
||||
}
|
||||
|
||||
func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
|
||||
res := make([]string, 0)
|
||||
for _, label := range namespace.Labels() {
|
||||
res = append(res, n.nodePrefix+_remove_hyphen(label))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (n *Neo4jRepository) Label(namespace types.NameSpace) string {
|
||||
labels := n.Labels(namespace)
|
||||
return strings.Join(labels, ":")
|
||||
}
|
||||
|
||||
// AddGraph implements interfaces.RetrieveGraphRepository.
|
||||
func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil
|
||||
}
|
||||
for _, graph := range graphs {
|
||||
if err := n.addGraph(ctx, namespace, graph); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error {
|
||||
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
|
||||
defer session.Close(ctx)
|
||||
|
||||
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
|
||||
node_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node
|
||||
SET node.chunks = apoc.coll.union(node.chunks, row.chunks)
|
||||
RETURN distinct 'done' AS result
|
||||
`
|
||||
nodeData := []map[string]interface{}{}
|
||||
for _, node := range graph.Node {
|
||||
nodeData = append(nodeData, map[string]interface{}{
|
||||
"name": node.Name,
|
||||
"knowledge_id": namespace.Knowledge,
|
||||
"props": map[string][]string{"attributes": node.Attributes},
|
||||
"chunks": node.Chunks,
|
||||
"labels": n.Labels(namespace),
|
||||
})
|
||||
}
|
||||
if _, err := tx.Run(ctx, node_import_query, map[string]interface{}{"data": nodeData}); err != nil {
|
||||
return nil, fmt.Errorf("failed to create nodes: %v", err)
|
||||
}
|
||||
|
||||
rel_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source
|
||||
CALL apoc.merge.node(row.target_labels, {name: row.target, kg: row.knowledge_id}, {}, {}) YIELD node as target
|
||||
CALL apoc.merge.relationship(source, row.type, {}, row.attributes, target) YIELD rel
|
||||
RETURN distinct 'done'
|
||||
`
|
||||
relData := []map[string]interface{}{}
|
||||
for _, rel := range graph.Relation {
|
||||
relData = append(relData, map[string]interface{}{
|
||||
"source": rel.Node1,
|
||||
"target": rel.Node2,
|
||||
"knowledge_id": namespace.Knowledge,
|
||||
"type": rel.Type,
|
||||
"source_labels": n.Labels(namespace),
|
||||
"target_labels": n.Labels(namespace),
|
||||
})
|
||||
}
|
||||
if _, err := tx.Run(ctx, rel_import_query, map[string]interface{}{"data": relData}); err != nil {
|
||||
return nil, fmt.Errorf("failed to create relationships: %v", err)
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to add graph: %v", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DelGraph implements interfaces.RetrieveGraphRepository.
|
||||
func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil
|
||||
}
|
||||
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
|
||||
defer session.Close(ctx)
|
||||
|
||||
result, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
|
||||
for _, namespace := range namespaces {
|
||||
labelExpr := n.Label(namespace)
|
||||
|
||||
deleteRelsQuery := `
|
||||
CALL apoc.periodic.iterate(
|
||||
"MATCH (n:` + labelExpr + ` {kg: $knowledge_id})-[r]-(m:` + labelExpr + ` {kg: $knowledge_id}) RETURN r",
|
||||
"DELETE r",
|
||||
{batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}}
|
||||
) YIELD batches, total
|
||||
RETURN total
|
||||
`
|
||||
if _, err := tx.Run(ctx, deleteRelsQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil {
|
||||
return nil, fmt.Errorf("failed to delete relationships: %v", err)
|
||||
}
|
||||
|
||||
deleteNodesQuery := `
|
||||
CALL apoc.periodic.iterate(
|
||||
"MATCH (n:` + labelExpr + ` {kg: $knowledge_id}) RETURN n",
|
||||
"DELETE n",
|
||||
{batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}}
|
||||
) YIELD batches, total
|
||||
RETURN total
|
||||
`
|
||||
if _, err := tx.Run(ctx, deleteNodesQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil {
|
||||
return nil, fmt.Errorf("failed to delete nodes: %v", err)
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof(ctx, "delete graph result: %v", result)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Neo4jRepository) SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil, nil
|
||||
}
|
||||
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead})
|
||||
defer session.Close(ctx)
|
||||
|
||||
result, err := session.ExecuteRead(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
|
||||
labelExpr := n.Label(namespace)
|
||||
query := `
|
||||
MATCH (n:` + labelExpr + `)-[r]-(m:` + labelExpr + `)
|
||||
WHERE ANY(nodeText IN $nodes WHERE n.name CONTAINS nodeText)
|
||||
RETURN n, r, m
|
||||
`
|
||||
params := map[string]interface{}{"nodes": nodes}
|
||||
result, err := tx.Run(ctx, query, params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to run query: %v", err)
|
||||
}
|
||||
|
||||
graphData := &types.GraphData{}
|
||||
nodeSeen := make(map[string]bool)
|
||||
for result.Next(ctx) {
|
||||
record := result.Record()
|
||||
node, _ := record.Get("n")
|
||||
rel, _ := record.Get("r")
|
||||
targetNode, _ := record.Get("m")
|
||||
|
||||
nodeData := node.(neo4j.Node)
|
||||
targetNodeData := targetNode.(neo4j.Node)
|
||||
|
||||
// Convert node to types.Node
|
||||
for _, n := range []neo4j.Node{nodeData, targetNodeData} {
|
||||
nameStr := n.Props["name"].(string)
|
||||
if _, ok := nodeSeen[nameStr]; !ok {
|
||||
nodeSeen[nameStr] = true
|
||||
graphData.Node = append(graphData.Node, &types.GraphNode{
|
||||
Name: nameStr,
|
||||
Chunks: listI2listS(n.Props["chunks"].([]interface{})),
|
||||
Attributes: listI2listS(n.Props["attributes"].([]interface{})),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Convert relationship to types.Relation
|
||||
relData := rel.(neo4j.Relationship)
|
||||
graphData.Relation = append(graphData.Relation, &types.GraphRelation{
|
||||
Node1: nodeData.Props["name"].(string),
|
||||
Node2: targetNodeData.Props["name"].(string),
|
||||
Type: relData.Type,
|
||||
})
|
||||
}
|
||||
return graphData, nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "search node failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
return result.(*types.GraphData), nil
|
||||
}
|
||||
|
||||
func listI2listS(list []any) []string {
|
||||
result := make([]string, len(list))
|
||||
for i, v := range list {
|
||||
result[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func mapI2mapS(prop map[string]any) map[string]string {
|
||||
attributes := make(map[string]string)
|
||||
for k, v := range prop {
|
||||
attributes[k] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
return attributes
|
||||
}
|
||||
@@ -0,0 +1,499 @@
|
||||
package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// PluginExtractEntity is a plugin for extracting entities from user queries
|
||||
// It uses historical dialog context and large language models to identify key entities in the user's original query
|
||||
type PluginExtractEntity struct {
|
||||
modelService interfaces.ModelService // Model service for calling large language models
|
||||
template *types.PromptTemplateStructured // Template for generating prompts
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
|
||||
}
|
||||
|
||||
// NewPluginRewrite creates a new query rewriting plugin instance
|
||||
// Also registers the plugin with the event manager
|
||||
func NewPluginExtractEntity(
|
||||
eventManager *EventManager,
|
||||
modelService interfaces.ModelService,
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
|
||||
config *config.Config,
|
||||
) *PluginExtractEntity {
|
||||
res := &PluginExtractEntity{
|
||||
modelService: modelService,
|
||||
template: config.ExtractManager.ExtractEntity,
|
||||
knowledgeBaseRepo: knowledgeBaseRepo,
|
||||
}
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
}
|
||||
|
||||
// ActivationEvents returns the list of event types this plugin responds to
|
||||
// This plugin only responds to REWRITE_QUERY events
|
||||
func (p *PluginExtractEntity) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.REWRITE_QUERY}
|
||||
}
|
||||
|
||||
// OnEvent processes triggered events
|
||||
// When receiving a REWRITE_QUERY event, it rewrites the user query using conversation history and the language model
|
||||
func (p *PluginExtractEntity) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Debugf(ctx, "skipping extract entity, neo4j is disabled")
|
||||
return next()
|
||||
}
|
||||
|
||||
query := chatManage.Query
|
||||
|
||||
model, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
}
|
||||
|
||||
kb, err := p.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chatManage.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
|
||||
return next()
|
||||
}
|
||||
if kb.ExtractConfig == nil {
|
||||
logger.Warnf(ctx, "failed to get extract config")
|
||||
return next()
|
||||
}
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: p.template.Description,
|
||||
Examples: p.template.Examples,
|
||||
}
|
||||
extractor := NewExtractor(model, template)
|
||||
graph, err := extractor.Extract(ctx, query)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to extract entities, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
}
|
||||
nodes := []string{}
|
||||
for _, node := range graph.Node {
|
||||
nodes = append(nodes, node.Name)
|
||||
}
|
||||
logger.Debugf(ctx, "extracted node: %v", nodes)
|
||||
chatManage.Entity = nodes
|
||||
return next()
|
||||
}
|
||||
|
||||
type Extractor struct {
|
||||
chat chat.Chat
|
||||
formater *Formater
|
||||
template *types.PromptTemplateStructured
|
||||
chatOpt *chat.ChatOptions
|
||||
}
|
||||
|
||||
func NewExtractor(
|
||||
chatModel chat.Chat,
|
||||
template *types.PromptTemplateStructured,
|
||||
) Extractor {
|
||||
think := false
|
||||
return Extractor{
|
||||
chat: chatModel,
|
||||
formater: NewFormater(),
|
||||
template: template,
|
||||
chatOpt: &chat.ChatOptions{
|
||||
Temperature: 0.3,
|
||||
MaxTokens: 4096,
|
||||
Thinking: &think,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) {
|
||||
generator := NewQAPromptGenerator(e.formater, e.template)
|
||||
|
||||
// logger.Debugf(ctx, "chat system: %s", generator.System(ctx))
|
||||
// logger.Debugf(ctx, "chat user: %s", generator.User(ctx, content))
|
||||
|
||||
chatResponse, err := e.chat.Chat(ctx, generator.Render(ctx, content), e.chatOpt)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to chat: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
graph, err := e.formater.ParseGraph(ctx, chatResponse.Content)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to parse graph: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
// e.RemoveUnknownRelation(ctx, graph)
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) {
|
||||
relationType := make(map[string]bool)
|
||||
for _, tag := range e.template.Tags {
|
||||
relationType[tag] = true
|
||||
}
|
||||
|
||||
relationNew := make([]*types.GraphRelation, 0)
|
||||
for _, relation := range graph.Relation {
|
||||
if _, ok := relationType[relation.Type]; ok {
|
||||
relationNew = append(relationNew, relation)
|
||||
} else {
|
||||
logger.Infof(ctx, "Unknown relation type %s with %v, ignore it", relation.Type, e.template.Tags)
|
||||
}
|
||||
}
|
||||
graph.Relation = relationNew
|
||||
}
|
||||
|
||||
type QAPromptGenerator struct {
|
||||
Formater *Formater
|
||||
Template *types.PromptTemplateStructured
|
||||
ExamplesHeading string
|
||||
QuestionHeading string
|
||||
QuestionPrefix string
|
||||
AnswerPrefix string
|
||||
}
|
||||
|
||||
func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator {
|
||||
return &QAPromptGenerator{
|
||||
Formater: formater,
|
||||
Template: template,
|
||||
ExamplesHeading: "# Examples",
|
||||
QuestionHeading: "# Question",
|
||||
QuestionPrefix: "Q: ",
|
||||
AnswerPrefix: "A: ",
|
||||
}
|
||||
}
|
||||
|
||||
func (qa *QAPromptGenerator) System(ctx context.Context) string {
|
||||
promptLines := []string{}
|
||||
|
||||
if len(qa.Template.Tags) == 0 {
|
||||
promptLines = append(promptLines, qa.Template.Description)
|
||||
} else {
|
||||
tags, _ := json.Marshal(qa.Template.Tags)
|
||||
promptLines = append(promptLines, fmt.Sprintf(qa.Template.Description, string(tags)))
|
||||
}
|
||||
if len(qa.Template.Examples) > 0 {
|
||||
promptLines = append(promptLines, qa.ExamplesHeading)
|
||||
for _, example := range qa.Template.Examples {
|
||||
// Question
|
||||
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, strings.TrimSpace(example.Text)))
|
||||
|
||||
// Answer
|
||||
answer, err := qa.Formater.formatExtraction(example.Node, example.Relation)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.AnswerPrefix, answer))
|
||||
|
||||
// new line
|
||||
promptLines = append(promptLines, "")
|
||||
}
|
||||
}
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
|
||||
promptLines := []string{}
|
||||
promptLines = append(promptLines, qa.QuestionHeading)
|
||||
promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, question))
|
||||
promptLines = append(promptLines, qa.AnswerPrefix)
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message {
|
||||
return []chat.Message{
|
||||
{
|
||||
Role: "system",
|
||||
Content: qa.System(ctx),
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: qa.User(ctx, question),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type FormatType string
|
||||
|
||||
const (
|
||||
FormatTypeJSON FormatType = "json"
|
||||
FormatTypeYAML FormatType = "yaml"
|
||||
)
|
||||
|
||||
const (
|
||||
_FENCE_START = "```"
|
||||
_LANGUAGE_TAG = `(?P<lang>[A-Za-z0-9_+-]+)?`
|
||||
_FENCE_NEWLINE = `(?:\s*\n)?`
|
||||
_FENCE_BODY = `(?P<body>[\s\S]*?)`
|
||||
_FENCE_END = "```"
|
||||
)
|
||||
|
||||
var _FENCE_RE = regexp.MustCompile(
|
||||
_FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END,
|
||||
)
|
||||
|
||||
type Formater struct {
|
||||
attributeSuffix string
|
||||
formatType FormatType
|
||||
useFences bool
|
||||
nodePrefix string
|
||||
|
||||
relationSource string
|
||||
relationTarget string
|
||||
relationPrefix string
|
||||
}
|
||||
|
||||
func NewFormater() *Formater {
|
||||
return &Formater{
|
||||
attributeSuffix: "_attributes",
|
||||
formatType: FormatTypeJSON,
|
||||
useFences: true,
|
||||
nodePrefix: "entity",
|
||||
relationSource: "entity1",
|
||||
relationTarget: "entity2",
|
||||
relationPrefix: "relation",
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) {
|
||||
items := make([]map[string]interface{}, 0)
|
||||
for _, node := range nodes {
|
||||
item := map[string]interface{}{
|
||||
f.nodePrefix: node.Name,
|
||||
}
|
||||
if len(node.Attributes) > 0 {
|
||||
item[fmt.Sprintf("%s%s", f.nodePrefix, f.attributeSuffix)] = node.Attributes
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
for _, relation := range relations {
|
||||
item := map[string]interface{}{
|
||||
f.relationSource: relation.Node1,
|
||||
f.relationTarget: relation.Node2,
|
||||
f.relationPrefix: relation.Type,
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
formatted := ""
|
||||
switch f.formatType {
|
||||
default:
|
||||
formattedBytes, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
formatted = string(formattedBytes)
|
||||
}
|
||||
if f.useFences {
|
||||
formatted = f.addFences(formatted)
|
||||
}
|
||||
return formatted, nil
|
||||
}
|
||||
|
||||
func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) {
|
||||
if text == "" {
|
||||
return nil, errors.New("Empty or invalid input string.")
|
||||
}
|
||||
content := f.extractContent(ctx, text)
|
||||
// logger.Debugf(ctx, "Extracted content: %s", content)
|
||||
if content == "" {
|
||||
return nil, errors.New("Empty or invalid input string.")
|
||||
}
|
||||
|
||||
var parsed interface{}
|
||||
var err error
|
||||
if f.formatType == FormatTypeJSON {
|
||||
err = json.Unmarshal([]byte(content), &parsed)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
|
||||
}
|
||||
if parsed == nil {
|
||||
return nil, fmt.Errorf("Content must be a list of extractions or a dict.")
|
||||
}
|
||||
|
||||
var items []interface{}
|
||||
if parsedMap, ok := parsed.(map[string]interface{}); ok {
|
||||
items = []interface{}{parsedMap}
|
||||
} else if parsedList, ok := parsed.([]interface{}); ok {
|
||||
items = parsedList
|
||||
} else {
|
||||
return nil, fmt.Errorf("Expected list or dict, got %T", parsed)
|
||||
}
|
||||
|
||||
itemsList := make([]map[string]interface{}, 0)
|
||||
for _, item := range items {
|
||||
if itemMap, ok := item.(map[string]interface{}); ok {
|
||||
itemsList = append(itemsList, itemMap)
|
||||
} else {
|
||||
return nil, fmt.Errorf("Each item in the sequence must be a mapping.")
|
||||
}
|
||||
}
|
||||
return itemsList, nil
|
||||
}
|
||||
|
||||
func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphData, error) {
|
||||
matchData, err := f.parseOutput(ctx, text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(matchData) == 0 {
|
||||
logger.Debugf(ctx, "Received empty extraction data.")
|
||||
return &types.GraphData{}, nil
|
||||
}
|
||||
// mm, _ := json.Marshal(matchData)
|
||||
// logger.Debugf(ctx, "Parsed graph data: %s", string(mm))
|
||||
|
||||
var nodes []*types.GraphNode
|
||||
var relations []*types.GraphRelation
|
||||
|
||||
for _, group := range matchData {
|
||||
switch {
|
||||
case group[f.nodePrefix] != nil:
|
||||
attributes := make([]string, 0)
|
||||
attributesKey := f.nodePrefix + f.attributeSuffix
|
||||
if attr, ok := group[attributesKey].([]interface{}); ok {
|
||||
for _, v := range attr {
|
||||
attributes = append(attributes, fmt.Sprintf("%v", v))
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, &types.GraphNode{
|
||||
Name: fmt.Sprintf("%v", group[f.nodePrefix]),
|
||||
Attributes: attributes,
|
||||
})
|
||||
case group[f.relationSource] != nil && group[f.relationTarget] != nil:
|
||||
relations = append(relations, &types.GraphRelation{
|
||||
Node1: fmt.Sprintf("%v", group[f.relationSource]),
|
||||
Node2: fmt.Sprintf("%v", group[f.relationTarget]),
|
||||
Type: fmt.Sprintf("%v", group[f.relationPrefix]),
|
||||
})
|
||||
default:
|
||||
logger.Warnf(ctx, "Unsupported graph group: %v", group)
|
||||
continue
|
||||
}
|
||||
}
|
||||
graph := &types.GraphData{
|
||||
Node: nodes,
|
||||
Relation: relations,
|
||||
}
|
||||
f.rebuildGraph(ctx, graph)
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) {
|
||||
nodeMap := make(map[string]*types.GraphNode)
|
||||
nodes := make([]*types.GraphNode, 0, len(graph.Node))
|
||||
for _, node := range graph.Node {
|
||||
if prenode, ok := nodeMap[node.Name]; ok {
|
||||
logger.Infof(ctx, "Duplicate node ID: %s, merge attribute", node.Name)
|
||||
// 修复panic:检查Attributes是否为nil
|
||||
if node.Attributes == nil {
|
||||
node.Attributes = make([]string, 0)
|
||||
}
|
||||
if prenode.Attributes != nil {
|
||||
for _, attr := range prenode.Attributes {
|
||||
node.Attributes = append(node.Attributes, attr)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
nodeMap[node.Name] = node
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
|
||||
relations := make([]*types.GraphRelation, 0, len(graph.Relation))
|
||||
for _, relation := range graph.Relation {
|
||||
if relation.Node1 == relation.Node2 {
|
||||
logger.Infof(ctx, "Duplicate relation, ignore it")
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := nodeMap[relation.Node1]; !ok {
|
||||
node := &types.GraphNode{Name: relation.Node1}
|
||||
nodes = append(nodes, node)
|
||||
nodeMap[relation.Node1] = node
|
||||
logger.Infof(ctx, "Add unknown source node ID: %s", relation.Node1)
|
||||
}
|
||||
if _, ok := nodeMap[relation.Node2]; !ok {
|
||||
node := &types.GraphNode{Name: relation.Node2}
|
||||
nodes = append(nodes, node)
|
||||
nodeMap[relation.Node2] = node
|
||||
logger.Infof(ctx, "Add unknown target node ID: %s", relation.Node2)
|
||||
}
|
||||
|
||||
relations = append(relations, relation)
|
||||
}
|
||||
*graph = types.GraphData{
|
||||
Node: nodes,
|
||||
Relation: relations,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Formater) extractContent(ctx context.Context, text string) string {
|
||||
if !f.useFences {
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
validTags := map[FormatType]map[string]struct{}{
|
||||
FormatTypeYAML: {"yaml": {}, "yml": {}},
|
||||
FormatTypeJSON: {"json": {}},
|
||||
}
|
||||
matches := _FENCE_RE.FindAllStringSubmatch(text, -1)
|
||||
var candidates []string
|
||||
for _, match := range matches {
|
||||
lang := match[1]
|
||||
body := match[2]
|
||||
if f.isValidLanguageTag(lang, validTags) {
|
||||
candidates = append(candidates, body)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(candidates) == 1:
|
||||
return strings.TrimSpace(candidates[0])
|
||||
|
||||
case len(candidates) > 1:
|
||||
logger.Warnf(ctx, "multiple candidates found: %d", len(candidates))
|
||||
return strings.TrimSpace(candidates[0])
|
||||
|
||||
case len(matches) == 1:
|
||||
logger.Debugf(ctx, "no candidate found, use first match without language tag: %s", matches[0][1])
|
||||
return strings.TrimSpace(matches[0][2])
|
||||
|
||||
case len(matches) > 1:
|
||||
logger.Warnf(ctx, "multiple matches found: %d", len(matches))
|
||||
return strings.TrimSpace(matches[0][2])
|
||||
|
||||
default:
|
||||
logger.Warnf(ctx, "no match found")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Formater) addFences(content string) string {
|
||||
content = strings.TrimSpace(content)
|
||||
return fmt.Sprintf("```%s\n%s\n```", f.formatType, content)
|
||||
}
|
||||
|
||||
func (f *Formater) isValidLanguageTag(lang string, validTags map[FormatType]map[string]struct{}) bool {
|
||||
if lang == "" {
|
||||
return true
|
||||
}
|
||||
tag := strings.TrimSpace(strings.ToLower(lang))
|
||||
validSet, ok := validTags[f.formatType]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, exists := validSet[tag]
|
||||
return exists
|
||||
}
|
||||
@@ -77,6 +77,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
}
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, searchResults...)
|
||||
}
|
||||
|
||||
// remove duplicate results
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// PluginSearch implements search functionality for chat pipeline
|
||||
type PluginSearchEntity struct {
|
||||
graphRepo interfaces.RetrieveGraphRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
}
|
||||
|
||||
func NewPluginSearchEntity(
|
||||
eventManager *EventManager,
|
||||
graphRepository interfaces.RetrieveGraphRepository,
|
||||
chunkRepository interfaces.ChunkRepository,
|
||||
knowledgeRepository interfaces.KnowledgeRepository,
|
||||
) *PluginSearchEntity {
|
||||
res := &PluginSearchEntity{
|
||||
graphRepo: graphRepository,
|
||||
chunkRepo: chunkRepository,
|
||||
knowledgeRepo: knowledgeRepository,
|
||||
}
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
}
|
||||
|
||||
// ActivationEvents returns the event types this plugin handles
|
||||
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.ENTITY_SEARCH}
|
||||
}
|
||||
|
||||
// OnEvent handles search events in the chat pipeline
|
||||
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
entity := chatManage.Entity
|
||||
if len(entity) == 0 {
|
||||
logger.Infof(ctx, "No entity found")
|
||||
return next()
|
||||
}
|
||||
|
||||
graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: chatManage.KnowledgeBaseID}, entity)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search node, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
}
|
||||
chatManage.GraphResult = graph
|
||||
logger.Infof(ctx, "search entity result count: %d", len(graph.Node))
|
||||
// graphStr, _ := json.Marshal(graph)
|
||||
// logger.Debugf(ctx, "search entity result: %s", string(graphStr))
|
||||
|
||||
chunkIDs := filterSeenChunk(ctx, graph, chatManage.SearchResult)
|
||||
if len(chunkIDs) == 0 {
|
||||
logger.Infof(ctx, "No new chunk found")
|
||||
return next()
|
||||
}
|
||||
chunks, err := p.chunkRepo.ListChunksByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), chunkIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to list chunks, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
}
|
||||
knowledgeIDs := []string{}
|
||||
for _, chunk := range chunks {
|
||||
knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID)
|
||||
}
|
||||
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledgeIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
}
|
||||
|
||||
knowledgeMap := map[string]*types.Knowledge{}
|
||||
for _, knowledge := range knowledges {
|
||||
knowledgeMap[knowledge.ID] = knowledge
|
||||
}
|
||||
for _, chunk := range chunks {
|
||||
searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID])
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, searchResult)
|
||||
}
|
||||
// remove duplicate results
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
if len(chatManage.SearchResult) == 0 {
|
||||
logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID)
|
||||
return ErrSearchNothing
|
||||
}
|
||||
logger.Infof(ctx, "search entity result count: %d, session_id: %s", len(chatManage.SearchResult), chatManage.SessionID)
|
||||
return next()
|
||||
}
|
||||
|
||||
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, chunk := range searchResult {
|
||||
seen[chunk.ID] = true
|
||||
}
|
||||
logger.Infof(ctx, "filterSeenChunk: seen count: %d", len(seen))
|
||||
|
||||
chunkIDs := []string{}
|
||||
for _, node := range graph.Node {
|
||||
for _, chunkID := range node.Chunks {
|
||||
if seen[chunkID] {
|
||||
continue
|
||||
}
|
||||
seen[chunkID] = true
|
||||
chunkIDs = append(chunkIDs, chunkID)
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "filterSeenChunk: new chunkIDs count: %d", len(chunkIDs))
|
||||
return chunkIDs
|
||||
}
|
||||
|
||||
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
|
||||
return &types.SearchResult{
|
||||
ID: chunk.ID,
|
||||
Content: chunk.Content,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
ChunkIndex: chunk.ChunkIndex,
|
||||
KnowledgeTitle: knowledge.Title,
|
||||
StartAt: chunk.StartAt,
|
||||
EndAt: chunk.EndAt,
|
||||
Seq: chunk.ChunkIndex,
|
||||
Score: 1.0,
|
||||
MatchType: types.MatchTypeGraph,
|
||||
Metadata: knowledge.GetMetadata(),
|
||||
ChunkType: string(chunk.ChunkType),
|
||||
ParentChunkID: chunk.ParentChunkID,
|
||||
ImageInfo: chunk.ImageInfo,
|
||||
KnowledgeFilename: knowledge.FileName,
|
||||
KnowledgeSource: knowledge.Source,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
func NewChunkExtractTask(ctx context.Context, client *asynq.Client, tenantID uint, chunkID string, modelID string) error {
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Debugf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil
|
||||
}
|
||||
payload, err := json.Marshal(types.ExtractChunkPayload{
|
||||
TenantID: tenantID,
|
||||
ChunkID: chunkID,
|
||||
ModelID: modelID,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.MaxRetry(3))
|
||||
info, err := client.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to enqueue task: %v", err)
|
||||
return fmt.Errorf("failed to enqueue task: %v", err)
|
||||
}
|
||||
logger.Infof(ctx, "enqueued task: id=%s queue=%s chunk=%s", info.ID, info.Queue, chunkID)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ChunkExtractService struct {
|
||||
template *types.PromptTemplateStructured
|
||||
modelService interfaces.ModelService
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
}
|
||||
|
||||
func NewChunkExtractService(
|
||||
config *config.Config,
|
||||
modelService interfaces.ModelService,
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
) interfaces.Extracter {
|
||||
generator := chatpipline.NewQAPromptGenerator(chatpipline.NewFormater(), config.ExtractManager.ExtractGraph)
|
||||
ctx := context.Background()
|
||||
logger.Debugf(ctx, "chunk extract system prompt: %s", generator.System(ctx))
|
||||
logger.Debugf(ctx, "chunk extract user prompt: %s", generator.User(ctx, "demo"))
|
||||
return &ChunkExtractService{
|
||||
template: config.ExtractManager.ExtractGraph,
|
||||
modelService: modelService,
|
||||
knowledgeBaseRepo: knowledgeBaseRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
graphEngine: graphEngine,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error {
|
||||
var p types.ExtractChunkPayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
logger.Errorf(ctx, "failed to unmarshal task payload: %v", err)
|
||||
return err
|
||||
}
|
||||
ctx = logger.WithRequestID(ctx, uuid.New().String())
|
||||
ctx = logger.WithField(ctx, "extract", p.ChunkID)
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, p.TenantID)
|
||||
|
||||
chunk, err := s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get chunk: %v", err)
|
||||
return err
|
||||
}
|
||||
kb, err := s.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chunk.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
|
||||
return err
|
||||
}
|
||||
if kb.ExtractConfig == nil {
|
||||
logger.Warnf(ctx, "failed to get extract config")
|
||||
return err
|
||||
}
|
||||
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get chat model: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: s.template.Description,
|
||||
Tags: kb.ExtractConfig.Tags,
|
||||
Examples: []types.GraphData{
|
||||
{
|
||||
Text: kb.ExtractConfig.Text,
|
||||
Node: kb.ExtractConfig.Nodes,
|
||||
Relation: kb.ExtractConfig.Relations,
|
||||
},
|
||||
},
|
||||
}
|
||||
extractor := chatpipline.NewExtractor(chatModel, template)
|
||||
graph, err := extractor.Extract(ctx, chunk.Content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk, err = s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "graph ignore chunk %s: %v", p.ChunkID, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, node := range graph.Node {
|
||||
node.Chunks = []string{chunk.ID}
|
||||
}
|
||||
if err = s.graphEngine.AddGraph(ctx,
|
||||
types.NameSpace{KnowledgeBase: chunk.KnowledgeBaseID, Knowledge: chunk.KnowledgeID},
|
||||
[]*types.GraphData{graph},
|
||||
); err != nil {
|
||||
logger.Errorf(ctx, "failed to add graph: %v", err)
|
||||
return err
|
||||
}
|
||||
// gg, _ := json.Marshal(graph)
|
||||
// logger.Infof(ctx, "extracted graph: %s", string(gg))
|
||||
return nil
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/client"
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -61,6 +62,8 @@ type knowledgeService struct {
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
fileSvc interfaces.FileService
|
||||
modelService interfaces.ModelService
|
||||
task *asynq.Client
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
}
|
||||
|
||||
// NewKnowledgeService creates a new knowledge service instance
|
||||
@@ -74,6 +77,8 @@ func NewKnowledgeService(
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
fileSvc interfaces.FileService,
|
||||
modelService interfaces.ModelService,
|
||||
task *asynq.Client,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
) (interfaces.KnowledgeService, error) {
|
||||
return &knowledgeService{
|
||||
config: config,
|
||||
@@ -85,6 +90,8 @@ func NewKnowledgeService(
|
||||
chunkRepo: chunkRepo,
|
||||
fileSvc: fileSvc,
|
||||
modelService: modelService,
|
||||
task: task,
|
||||
graphEngine: graphEngine,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -488,6 +495,16 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
return nil
|
||||
})
|
||||
|
||||
// Delete the knowledge graph
|
||||
wg.Go(func() error {
|
||||
namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}
|
||||
if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err = wg.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -561,6 +578,19 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
return nil
|
||||
})
|
||||
|
||||
// Delete the knowledge graph
|
||||
wg.Go(func() error {
|
||||
namespaces := []types.NameSpace{}
|
||||
for _, knowledge := range knowledgeList {
|
||||
namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID})
|
||||
}
|
||||
if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err = wg.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1158,6 +1188,15 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList))
|
||||
|
||||
logger.Infof(ctx, "processChunks create relationship rag task")
|
||||
for _, chunk := range textChunks {
|
||||
err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed")
|
||||
span.RecordError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update knowledge status to completed
|
||||
knowledge.ParseStatus = "completed"
|
||||
knowledge.EnableStatus = "enabled"
|
||||
|
||||
@@ -2,12 +2,11 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"encoding/json"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/common"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -376,8 +375,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
// processSearchResults handles the processing of search results, optimizing database queries
|
||||
func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
|
||||
chunks []*types.IndexWithScore) ([]*types.SearchResult, error) {
|
||||
|
||||
chunks []*types.IndexWithScore,
|
||||
) ([]*types.SearchResult, error) {
|
||||
if len(chunks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -527,8 +526,8 @@ func (s *knowledgeBaseService) collectRelatedChunkIDs(chunk *types.Chunk, proces
|
||||
func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk,
|
||||
knowledge *types.Knowledge,
|
||||
score float64,
|
||||
matchType types.MatchType) *types.SearchResult {
|
||||
|
||||
matchType types.MatchType,
|
||||
) *types.SearchResult {
|
||||
return &types.SearchResult{
|
||||
ID: chunk.ID,
|
||||
Content: chunk.Content,
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// client is the global asyncq client instance
|
||||
var client *asynq.Client
|
||||
|
||||
// InitAsyncq initializes the asyncq client with configuration
|
||||
// It creates a new client and starts the server in a goroutine
|
||||
func InitAsyncq(config *config.Config) error {
|
||||
cfg := config.Asynq
|
||||
client = asynq.NewClient(asynq.RedisClientOpt{
|
||||
Addr: cfg.Addr,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
ReadTimeout: cfg.ReadTimeout,
|
||||
WriteTimeout: cfg.WriteTimeout,
|
||||
})
|
||||
go run(cfg)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAsyncqClient returns the global asyncq client instance
|
||||
func GetAsyncqClient() *asynq.Client {
|
||||
return client
|
||||
}
|
||||
|
||||
// handleFunc stores registered task handlers
|
||||
var handleFunc = map[string]asynq.HandlerFunc{}
|
||||
|
||||
// RegisterHandlerFunc registers a handler function for a specific task type
|
||||
func RegisterHandlerFunc(taskType string, handlerFunc asynq.HandlerFunc) {
|
||||
handleFunc[taskType] = handlerFunc
|
||||
}
|
||||
|
||||
// run starts the asyncq server with the given configuration
|
||||
// It creates a new server, sets up handlers, and runs the server
|
||||
func run(config *config.AsynqConfig) {
|
||||
srv := asynq.NewServer(
|
||||
asynq.RedisClientOpt{
|
||||
Addr: config.Addr,
|
||||
Username: config.Username,
|
||||
Password: config.Password,
|
||||
ReadTimeout: config.ReadTimeout,
|
||||
WriteTimeout: config.WriteTimeout,
|
||||
},
|
||||
asynq.Config{
|
||||
Concurrency: config.Concurrency,
|
||||
Queues: map[string]int{
|
||||
"critical": 6, // Highest priority queue
|
||||
"default": 3, // Default priority queue
|
||||
"low": 1, // Lowest priority queue
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
// Create a new mux and register all handlers
|
||||
mux := asynq.NewServeMux()
|
||||
for typ, handler := range handleFunc {
|
||||
mux.HandleFunc(typ, handler)
|
||||
}
|
||||
|
||||
// Start the server
|
||||
if err := srv.Run(mux); err != nil {
|
||||
log.Fatalf("could not run server: %v", err)
|
||||
}
|
||||
}
|
||||
+14
-10
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/go-viper/mapstructure/v2"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
@@ -18,10 +19,10 @@ type Config struct {
|
||||
KnowledgeBase *KnowledgeBaseConfig `yaml:"knowledge_base" json:"knowledge_base"`
|
||||
Tenant *TenantConfig `yaml:"tenant" json:"tenant"`
|
||||
Models []ModelConfig `yaml:"models" json:"models"`
|
||||
Asynq *AsynqConfig `yaml:"asynq" json:"asynq"`
|
||||
VectorDatabase *VectorDatabaseConfig `yaml:"vector_database" json:"vector_database"`
|
||||
DocReader *DocReaderConfig `yaml:"docreader" json:"docreader"`
|
||||
StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"`
|
||||
ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"`
|
||||
}
|
||||
|
||||
type DocReaderConfig struct {
|
||||
@@ -109,15 +110,6 @@ type ModelConfig struct {
|
||||
Parameters map[string]interface{} `yaml:"parameters" json:"parameters"`
|
||||
}
|
||||
|
||||
type AsynqConfig struct {
|
||||
Addr string `yaml:"addr" json:"addr"`
|
||||
Username string `yaml:"username" json:"username"`
|
||||
Password string `yaml:"password" json:"password"`
|
||||
ReadTimeout time.Duration `yaml:"read_timeout" json:"read_timeout"`
|
||||
WriteTimeout time.Duration `yaml:"write_timeout" json:"write_timeout"`
|
||||
Concurrency int `yaml:"concurrency" json:"concurrency"`
|
||||
}
|
||||
|
||||
// StreamManagerConfig 流管理器配置
|
||||
type StreamManagerConfig struct {
|
||||
Type string `yaml:"type" json:"type"` // 类型: "memory" 或 "redis"
|
||||
@@ -134,6 +126,18 @@ type RedisConfig struct {
|
||||
TTL time.Duration `yaml:"ttl" json:"ttl"` // 过期时间(小时)
|
||||
}
|
||||
|
||||
// ExtractManagerConfig 抽取管理器配置
|
||||
type ExtractManagerConfig struct {
|
||||
ExtractGraph *types.PromptTemplateStructured `yaml:"extract_graph" json:"extract_graph"`
|
||||
ExtractEntity *types.PromptTemplateStructured `yaml:"extract_entity" json:"extract_entity"`
|
||||
FabriText *FebriText `yaml:"fabri_text" json:"fabri_text"`
|
||||
}
|
||||
|
||||
type FebriText struct {
|
||||
WithTag string `yaml:"with_tag" json:"with_tag"`
|
||||
WithNoTag string `yaml:"with_no_tag" json:"with_no_tag"`
|
||||
}
|
||||
|
||||
// LoadConfig 从配置文件加载配置
|
||||
func LoadConfig() (*Config, error) {
|
||||
// 设置配置文件名和路径
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"go.uber.org/dig"
|
||||
"gorm.io/driver/postgres"
|
||||
@@ -22,6 +23,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
@@ -68,6 +70,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
// External service clients
|
||||
must(container.Provide(initDocReaderClient))
|
||||
must(container.Provide(initOllamaService))
|
||||
must(container.Provide(initNeo4jClient))
|
||||
must(container.Provide(stream.NewStreamManager))
|
||||
|
||||
// Data repositories layer
|
||||
@@ -80,6 +83,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(repository.NewModelRepository))
|
||||
must(container.Provide(repository.NewUserRepository))
|
||||
must(container.Provide(repository.NewAuthTokenRepository))
|
||||
must(container.Provide(neo4jRepo.NewNeo4jRepository))
|
||||
|
||||
// Business service layer
|
||||
must(container.Provide(service.NewTenantService))
|
||||
@@ -93,6 +97,7 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(service.NewDatasetService))
|
||||
must(container.Provide(service.NewEvaluationService))
|
||||
must(container.Provide(service.NewUserService))
|
||||
must(container.Provide(service.NewChunkExtractService))
|
||||
|
||||
// Chat pipeline components for processing chat requests
|
||||
must(container.Provide(chatpipline.NewEventManager))
|
||||
@@ -107,6 +112,8 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Invoke(chatpipline.NewPluginFilterTopK))
|
||||
must(container.Invoke(chatpipline.NewPluginPreprocess))
|
||||
must(container.Invoke(chatpipline.NewPluginRewrite))
|
||||
must(container.Invoke(chatpipline.NewPluginExtractEntity))
|
||||
must(container.Invoke(chatpipline.NewPluginSearchEntity))
|
||||
|
||||
// HTTP handlers layer
|
||||
must(container.Provide(handler.NewTenantHandler))
|
||||
@@ -123,6 +130,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
|
||||
// Router configuration
|
||||
must(container.Provide(router.NewRouter))
|
||||
must(container.Provide(router.NewAsyncqClient))
|
||||
must(container.Provide(router.NewAsynqServer))
|
||||
must(container.Invoke(router.RunAsynqServer))
|
||||
|
||||
return container
|
||||
}
|
||||
@@ -184,6 +194,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
err = db.AutoMigrate(
|
||||
&types.User{},
|
||||
&types.AuthToken{},
|
||||
&types.KnowledgeBase{},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to auto-migrate database tables: %v", err)
|
||||
@@ -385,3 +396,23 @@ func initOllamaService() (*ollama.OllamaService, error) {
|
||||
// Get Ollama service from existing factory function
|
||||
return ollama.GetOllamaService()
|
||||
}
|
||||
|
||||
func initNeo4jClient() (neo4j.Driver, error) {
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Debugf(context.Background(), "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil, nil
|
||||
}
|
||||
uri := os.Getenv("NEO4J_URI")
|
||||
username := os.Getenv("NEO4J_USERNAME")
|
||||
password := os.Getenv("NEO4J_PASSWORD")
|
||||
|
||||
driver, err := neo4j.NewDriver(uri, neo4j.BasicAuth(username, password, ""))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = driver.VerifyAuthentication(context.Background(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,15 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"strconv"
|
||||
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -133,6 +134,21 @@ type InitializationRequest struct {
|
||||
ChunkOverlap int `json:"chunkOverlap" binding:"min=0"`
|
||||
Separators []string `json:"separators" binding:"required,min=1"`
|
||||
} `json:"documentSplitting" binding:"required"`
|
||||
|
||||
NodeExtract struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Text string `json:"text"`
|
||||
Tags []string `json:"tags"`
|
||||
Nodes []struct {
|
||||
Name string `json:"name"`
|
||||
Attributes []string `json:"attributes"`
|
||||
} `json:"nodes"`
|
||||
Relations []struct {
|
||||
Node1 string `json:"node1"`
|
||||
Node2 string `json:"node2"`
|
||||
Type string `json:"type"`
|
||||
} `json:"relations"`
|
||||
} `json:"nodeExtract"`
|
||||
}
|
||||
|
||||
// InitializeByKB 根据知识库ID执行配置更新
|
||||
@@ -207,6 +223,25 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// 验证Node Extractor配置(如果启用)
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" && req.NodeExtract.Enabled {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE"))
|
||||
return
|
||||
}
|
||||
if req.NodeExtract.Enabled {
|
||||
if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("Node Extractor配置不完整"))
|
||||
return
|
||||
}
|
||||
if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("请先提取实体和关系"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 处理模型创建/更新
|
||||
modelsToProcess := []struct {
|
||||
modelType types.ModelType
|
||||
@@ -406,6 +441,29 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
kb.StorageConfig = types.StorageConfig{}
|
||||
}
|
||||
|
||||
if req.NodeExtract.Enabled {
|
||||
kb.ExtractConfig = &types.ExtractConfig{
|
||||
Text: req.NodeExtract.Text,
|
||||
Tags: req.NodeExtract.Tags,
|
||||
Nodes: make([]*types.GraphNode, 0),
|
||||
Relations: make([]*types.GraphRelation, 0),
|
||||
}
|
||||
for _, rnode := range req.NodeExtract.Nodes {
|
||||
node := &types.GraphNode{
|
||||
Name: rnode.Name,
|
||||
Attributes: rnode.Attributes,
|
||||
}
|
||||
kb.ExtractConfig.Nodes = append(kb.ExtractConfig.Nodes, node)
|
||||
}
|
||||
for _, relation := range req.NodeExtract.Relations {
|
||||
kb.ExtractConfig.Relations = append(kb.ExtractConfig.Relations, &types.GraphRelation{
|
||||
Node1: relation.Node1,
|
||||
Node2: relation.Node2,
|
||||
Type: relation.Type,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr})
|
||||
@@ -710,7 +768,6 @@ func (h *InitializationHandler) downloadModelAsync(ctx context.Context,
|
||||
err := h.pullModelWithProgress(ctx, modelName, func(progress float64, message string) {
|
||||
h.updateTaskStatus(taskID, "downloading", progress, message)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_name": modelName,
|
||||
@@ -777,7 +834,6 @@ func (h *InitializationHandler) pullModelWithProgress(ctx context.Context,
|
||||
)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to pull model: %w", err)
|
||||
}
|
||||
@@ -971,6 +1027,20 @@ func buildConfigResponse(models []*types.Model,
|
||||
}
|
||||
}
|
||||
|
||||
if kb.ExtractConfig != nil {
|
||||
config["nodeExtract"] = map[string]interface{}{
|
||||
"enabled": true,
|
||||
"text": kb.ExtractConfig.Text,
|
||||
"tags": kb.ExtractConfig.Tags,
|
||||
"nodes": kb.ExtractConfig.Nodes,
|
||||
"relations": kb.ExtractConfig.Relations,
|
||||
}
|
||||
} else {
|
||||
config["nodeExtract"] = map[string]interface{}{
|
||||
"enabled": false,
|
||||
}
|
||||
}
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
@@ -1148,8 +1218,8 @@ func (h *InitializationHandler) checkRemoteModelConnection(ctx context.Context,
|
||||
|
||||
// checkRerankModelConnection 检查Rerank模型连接和功能的内部方法
|
||||
func (h *InitializationHandler) checkRerankModelConnection(ctx context.Context,
|
||||
modelName, baseURL, apiKey string) (bool, string) {
|
||||
|
||||
modelName, baseURL, apiKey string,
|
||||
) (bool, string) {
|
||||
// 创建Reranker配置
|
||||
config := &rerank.RerankerConfig{
|
||||
APIKey: apiKey,
|
||||
@@ -1492,3 +1562,214 @@ func (h *InitializationHandler) testMultimodalWithDocReader(
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// TextRelationExtractionRequest 文本关系提取请求结构
|
||||
type TextRelationExtractionRequest struct {
|
||||
Text string `json:"text" binding:"required"`
|
||||
Tags []string `json:"tags" binding:"required"`
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
Source string `json:"source"`
|
||||
ModelName string `json:"modelName"`
|
||||
BaseUrl string `json:"baseUrl"`
|
||||
ApiKey string `json:"apiKey"`
|
||||
}
|
||||
|
||||
// TextRelationExtractionResponse 文本关系提取响应结构
|
||||
type TextRelationExtractionResponse struct {
|
||||
Nodes []*types.GraphNode `json:"nodes"`
|
||||
Relations []*types.GraphRelation `json:"relations"`
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) ExtractTextRelations(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req TextRelationExtractionRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
logger.Error(ctx, "文本关系提取请求参数错误")
|
||||
c.Error(errors.NewBadRequestError("文本关系提取请求参数错误"))
|
||||
return
|
||||
}
|
||||
|
||||
// 验证文本内容
|
||||
if len(req.Text) == 0 {
|
||||
c.Error(errors.NewBadRequestError("文本内容不能为空"))
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.Text) > 5000 {
|
||||
c.Error(errors.NewBadRequestError("文本内容长度不能超过5000字符"))
|
||||
return
|
||||
}
|
||||
|
||||
// 验证标签
|
||||
if len(req.Tags) == 0 {
|
||||
c.Error(errors.NewBadRequestError("至少需要选择一个关系标签"))
|
||||
return
|
||||
}
|
||||
|
||||
// 调用模型服务进行文本关系提取
|
||||
result, err := h.extractRelationsFromText(ctx, req.Text, req.Tags, req.LLMConfig)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "文本关系提取失败", err)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": gin.H{
|
||||
"success": false,
|
||||
"message": err.Error(),
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": result,
|
||||
})
|
||||
}
|
||||
|
||||
// extractRelationsFromText 从文本中提取关系
|
||||
func (h *InitializationHandler) extractRelationsFromText(ctx context.Context, text string, tags []string, llm LLMConfig) (*TextRelationExtractionResponse, error) {
|
||||
chatModel, err := chat.NewChat(&chat.ChatConfig{
|
||||
ModelID: "initialization",
|
||||
APIKey: llm.ApiKey,
|
||||
BaseURL: llm.BaseUrl,
|
||||
ModelName: llm.ModelName,
|
||||
Source: types.ModelSource(llm.Source),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx, "初始化模型服务失败", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
template := &types.PromptTemplateStructured{
|
||||
Description: h.config.ExtractManager.ExtractGraph.Description,
|
||||
Tags: tags,
|
||||
Examples: h.config.ExtractManager.ExtractGraph.Examples,
|
||||
}
|
||||
|
||||
extractor := chatpipline.NewExtractor(chatModel, template)
|
||||
graph, err := extractor.Extract(ctx, text)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "文本关系提取失败", err)
|
||||
return nil, err
|
||||
}
|
||||
extractor.RemoveUnknownRelation(ctx, graph)
|
||||
|
||||
result := &TextRelationExtractionResponse{
|
||||
Nodes: graph.Node,
|
||||
Relations: graph.Relation,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type FabriTextRequest struct {
|
||||
Tags []string `json:"tags"`
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
type FabriTextResponse struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) FabriText(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
var req FabriTextRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
logger.Error(ctx, "生成示例文本请求参数错误")
|
||||
c.Error(errors.NewBadRequestError("生成示例文本请求参数错误"))
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.fabriText(ctx, req.Tags, req.LLMConfig)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "生成示例文本失败", err)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": gin.H{
|
||||
"success": false,
|
||||
"message": err.Error(),
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": FabriTextResponse{Text: result},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, llm LLMConfig) (string, error) {
|
||||
chatModel, err := chat.NewChat(&chat.ChatConfig{
|
||||
ModelID: "initialization",
|
||||
APIKey: llm.ApiKey,
|
||||
BaseURL: llm.BaseUrl,
|
||||
ModelName: llm.ModelName,
|
||||
Source: types.ModelSource(llm.Source),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx, "初始化模型服务失败", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
content := h.config.ExtractManager.FabriText.WithNoTag
|
||||
if len(tags) > 0 {
|
||||
tagStr, _ := json.Marshal(tags)
|
||||
content = fmt.Sprintf(h.config.ExtractManager.FabriText.WithTag, string(tagStr))
|
||||
}
|
||||
|
||||
think := false
|
||||
result, err := chatModel.Chat(ctx, []chat.Message{
|
||||
{Role: "user", Content: content},
|
||||
}, &chat.ChatOptions{
|
||||
Temperature: 0.3,
|
||||
MaxTokens: 4096,
|
||||
Thinking: &think,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx, "生成示例文本失败", err)
|
||||
return "", err
|
||||
}
|
||||
return result.Content, nil
|
||||
}
|
||||
|
||||
type FabriTagRequest struct {
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
type FabriTagResponse struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
|
||||
var tagOptions = []string{
|
||||
"内容", "文化", "人物", "事件", "时间", "地点", "作品", "作者", "关系", "属性",
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) FabriTag(c *gin.Context) {
|
||||
tagRandom := RandomSelect(tagOptions, rand.Intn(len(tagOptions)-1)+1)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": FabriTagResponse{Tags: tagRandom},
|
||||
})
|
||||
}
|
||||
|
||||
func RandomSelect(strs []string, n int) []string {
|
||||
if n <= 0 {
|
||||
return []string{}
|
||||
}
|
||||
result := make([]string, len(strs))
|
||||
copy(result, strs)
|
||||
rand.Shuffle(len(result), func(i, j int) {
|
||||
result[i], result[j] = result[j], result[i]
|
||||
})
|
||||
|
||||
if n > len(strs) {
|
||||
n = len(strs)
|
||||
}
|
||||
return result[:n]
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ type CreateSessionRequest struct {
|
||||
func (h *SessionHandler) CreateSession(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation)
|
||||
// logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation)
|
||||
|
||||
// Parse and validate the request body
|
||||
var request CreateSessionRequest
|
||||
|
||||
@@ -267,6 +267,10 @@ func RegisterInitializationRoutes(r *gin.RouterGroup, handler *handler.Initializ
|
||||
r.POST("/initialization/embedding/test", handler.TestEmbeddingModel)
|
||||
r.POST("/initialization/rerank/check", handler.CheckRerankModel)
|
||||
r.POST("/initialization/multimodal/test", handler.TestMultimodalFunction)
|
||||
|
||||
r.POST("/initialization/extract/text-relation", handler.ExtractTextRelations)
|
||||
r.POST("/initialization/extract/fabri-tag", handler.FabriTag)
|
||||
r.POST("/initialization/extract/fabri-text", handler.FabriText)
|
||||
}
|
||||
|
||||
// RegisterSystemRoutes registers system information routes
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"go.uber.org/dig"
|
||||
)
|
||||
|
||||
type AsynqTaskParams struct {
|
||||
dig.In
|
||||
|
||||
Server *asynq.Server
|
||||
Extracter interfaces.Extracter
|
||||
}
|
||||
|
||||
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
opt := &asynq.RedisClientOpt{
|
||||
Addr: os.Getenv("REDIS_ADDR"),
|
||||
Password: os.Getenv("REDIS_PASSWORD"),
|
||||
ReadTimeout: 100 * time.Millisecond,
|
||||
WriteTimeout: 200 * time.Millisecond,
|
||||
DB: 0,
|
||||
}
|
||||
return opt
|
||||
}
|
||||
|
||||
func NewAsyncqClient() *asynq.Client {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
client := asynq.NewClient(opt)
|
||||
return client
|
||||
}
|
||||
|
||||
func NewAsynqServer() *asynq.Server {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
srv := asynq.NewServer(
|
||||
opt,
|
||||
asynq.Config{
|
||||
Queues: map[string]int{
|
||||
"critical": 6, // Highest priority queue
|
||||
"default": 3, // Default priority queue
|
||||
"low": 1, // Lowest priority queue
|
||||
},
|
||||
},
|
||||
)
|
||||
return srv
|
||||
}
|
||||
|
||||
func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Create a new mux and register all handlers
|
||||
mux := asynq.NewServeMux()
|
||||
|
||||
mux.HandleFunc(types.TypeChunkExtract, params.Extracter.Extract)
|
||||
|
||||
go func() {
|
||||
// Start the server
|
||||
if err := params.Server.Run(mux); err != nil {
|
||||
log.Fatalf("could not run server: %v", err)
|
||||
}
|
||||
}()
|
||||
return mux
|
||||
}
|
||||
@@ -28,6 +28,8 @@ type ChatManage struct {
|
||||
SearchResult []*SearchResult `json:"-"` // Results from search phase
|
||||
RerankResult []*SearchResult `json:"-"` // Results after reranking
|
||||
MergeResult []*SearchResult `json:"-"` // Final merged results after all processing
|
||||
Entity []string `json:"-"` // List of identified entities
|
||||
GraphResult *GraphData `json:"-"` // Graph data from search phase
|
||||
UserContent string `json:"-"` // Processed user content
|
||||
ChatResponse *ChatResponse `json:"-"` // Final response from chat model
|
||||
ResponseChan <-chan StreamResponse `json:"-"` // Channel for streaming responses
|
||||
@@ -75,6 +77,7 @@ const (
|
||||
PREPROCESS_QUERY EventType = "preprocess_query" // Query preprocessing stage
|
||||
REWRITE_QUERY EventType = "rewrite_query" // Query rewriting for better retrieval
|
||||
CHUNK_SEARCH EventType = "chunk_search" // Search for relevant chunks
|
||||
ENTITY_SEARCH EventType = "entity_search" // Search for relevant entities
|
||||
CHUNK_RERANK EventType = "chunk_rerank" // Rerank search results
|
||||
CHUNK_MERGE EventType = "chunk_merge" // Merge similar chunks
|
||||
INTO_CHAT_MESSAGE EventType = "into_chat_message" // Convert chunks into chat messages
|
||||
@@ -104,6 +107,7 @@ var Pipline = map[string][]EventType{
|
||||
REWRITE_QUERY,
|
||||
PREPROCESS_QUERY,
|
||||
CHUNK_SEARCH,
|
||||
ENTITY_SEARCH,
|
||||
CHUNK_RERANK,
|
||||
CHUNK_MERGE,
|
||||
FILTER_TOP_K,
|
||||
|
||||
@@ -19,6 +19,7 @@ const (
|
||||
MatchTypeHistory
|
||||
MatchTypeParentChunk // 父Chunk匹配类型
|
||||
MatchTypeRelationChunk // 关系Chunk匹配类型
|
||||
MatchTypeGraph
|
||||
)
|
||||
|
||||
// IndexInfo contains information about indexed content
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package types
|
||||
|
||||
const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
)
|
||||
|
||||
type ExtractChunkPayload struct {
|
||||
TenantID uint `json:"tenant_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ModelID string `json:"model_id"`
|
||||
}
|
||||
|
||||
type PromptTemplateStructured struct {
|
||||
Description string `json:"description"`
|
||||
Tags []string `json:"tags"`
|
||||
Examples []GraphData `json:"examples"`
|
||||
}
|
||||
|
||||
type GraphNode struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Chunks []string `json:"chunks,omitempty"`
|
||||
Attributes []string `json:"attributes,omitempty"`
|
||||
}
|
||||
|
||||
type GraphRelation struct {
|
||||
Node1 string `json:"node1,omitempty"`
|
||||
Node2 string `json:"node2,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type GraphData struct {
|
||||
Text string `json:"text,omitempty"`
|
||||
Node []*GraphNode `json:"node,omitempty"`
|
||||
Relation []*GraphRelation `json:"relation,omitempty"`
|
||||
}
|
||||
|
||||
type NameSpace struct {
|
||||
KnowledgeBase string `json:"knowledge_base"`
|
||||
Knowledge string `json:"knowledge"`
|
||||
}
|
||||
|
||||
func (n NameSpace) Labels() []string {
|
||||
res := make([]string, 0)
|
||||
if n.KnowledgeBase != "" {
|
||||
res = append(res, n.KnowledgeBase)
|
||||
}
|
||||
if n.Knowledge != "" {
|
||||
res = append(res, n.Knowledge)
|
||||
}
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
type Extracter interface {
|
||||
Extract(ctx context.Context, t *asynq.Task) error
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type RetrieveGraphRepository interface {
|
||||
AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error
|
||||
DelGraph(ctx context.Context, namespace []types.NameSpace) error
|
||||
SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error)
|
||||
}
|
||||
@@ -38,6 +38,8 @@ type KnowledgeBase struct {
|
||||
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
|
||||
// Storage config
|
||||
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
|
||||
// Extract config
|
||||
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
|
||||
// Creation time of the knowledge base
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time of the knowledge base
|
||||
@@ -167,3 +169,27 @@ func (c *VLMConfig) Scan(value interface{}) error {
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
type ExtractConfig struct {
|
||||
Text string `yaml:"text" json:"text"`
|
||||
Tags []string `yaml:"tags" json:"tags"`
|
||||
Nodes []*GraphNode `yaml:"nodes" json:"nodes"`
|
||||
Relations []*GraphRelation `yaml:"relations" json:"relations"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert ExtractConfig to database value
|
||||
func (e ExtractConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface, used to convert database value to ExtractConfig
|
||||
func (e *ExtractConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, e)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user