refactor: Remove agent_test command-line tool and associated files to streamline project structure and eliminate unused components

This commit is contained in:
wizardchen
2025-12-01 17:24:57 +08:00
parent 60241a2d29
commit 93af4460d6
40 changed files with 387 additions and 1692 deletions
@@ -11,19 +11,23 @@ import (
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
)
// Neo4jRepository is a repository for Neo4j
type Neo4jRepository struct {
driver neo4j.Driver
nodePrefix string
}
// NewNeo4jRepository creates a new Neo4j repository
func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository {
return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"}
}
// _remove_hyphen removes hyphens from a string
func _remove_hyphen(s string) string {
return strings.ReplaceAll(s, "-", "_")
}
// Labels returns the labels for a namespace
func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
res := make([]string, 0)
for _, label := range namespace.Labels() {
@@ -32,12 +36,13 @@ func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
return res
}
// Label returns the label for a namespace
func (n *Neo4jRepository) Label(namespace types.NameSpace) string {
labels := n.Labels(namespace)
return strings.Join(labels, ":")
}
// AddGraph implements interfaces.RetrieveGraphRepository.
// AddGraph adds a graph to the Neo4j repository
func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error {
if n.driver == nil {
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
@@ -51,11 +56,13 @@ func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpac
return nil
}
// addGraph adds a graph to the Neo4j repository
func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error {
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close(ctx)
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
// Node import query
node_import_query := `
UNWIND $data AS row
CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node
@@ -76,6 +83,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
return nil, fmt.Errorf("failed to create nodes: %v", err)
}
// Relationship import query
rel_import_query := `
UNWIND $data AS row
CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source
@@ -106,7 +114,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
return nil
}
// DelGraph implements interfaces.RetrieveGraphRepository.
// DelGraph deletes a graph from the Neo4j repository
func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error {
if n.driver == nil {
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
@@ -152,6 +160,7 @@ func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameS
return nil
}
// SearchNode searches for nodes in the Neo4j repository
func (n *Neo4jRepository) SearchNode(
ctx context.Context,
namespace types.NameSpace,
+7 -1
View File
@@ -8,6 +8,7 @@ import (
"gorm.io/gorm"
)
// knowledgeTagRepository is a repository for knowledge tags
type knowledgeTagRepository struct {
db *gorm.DB
}
@@ -17,14 +18,17 @@ func NewKnowledgeTagRepository(db *gorm.DB) interfaces.KnowledgeTagRepository {
return &knowledgeTagRepository{db: db}
}
// Create creates a new knowledge tag
func (r *knowledgeTagRepository) Create(ctx context.Context, tag *types.KnowledgeTag) error {
return r.db.WithContext(ctx).Create(tag).Error
}
// Update updates a knowledge tag
func (r *knowledgeTagRepository) Update(ctx context.Context, tag *types.KnowledgeTag) error {
return r.db.WithContext(ctx).Save(tag).Error
}
// GetByID gets a knowledge tag by ID
func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.KnowledgeTag, error) {
var tag types.KnowledgeTag
if err := r.db.WithContext(ctx).
@@ -35,6 +39,7 @@ func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, i
return &tag, nil
}
// ListByKB lists knowledge tags by knowledge base ID
func (r *knowledgeTagRepository) ListByKB(
ctx context.Context,
tenantID uint64,
@@ -50,13 +55,14 @@ func (r *knowledgeTagRepository) ListByKB(
return tags, nil
}
// Delete deletes a knowledge tag
func (r *knowledgeTagRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
return r.db.WithContext(ctx).
Where("tenant_id = ? AND id = ?", tenantID, id).
Delete(&types.KnowledgeTag{}).Error
}
// CountReferences returns how many knowledges and chunks reference this tag.
// CountReferences returns the number of knowledges and chunks that reference this tag
func (r *knowledgeTagRepository) CountReferences(
ctx context.Context,
tenantID uint64,
@@ -10,31 +10,32 @@ import (
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// pipelineInfo logs pipeline info level entries.
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineInfo(ctx, stage, action, fields)
}
// pipelineWarn logs pipeline warning level entries.
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineWarn(ctx, stage, action, fields)
}
// pipelineError logs pipeline error level entries.
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineError(ctx, stage, action, fields)
}
// prepareChatModel shared logic to prepare chat model and options
// it gets the chat model and sets up the chat options based on the chat manage.
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
chatManage *types.ChatManage,
) (chat.Chat, *chat.ChatOptions, error) {
logger.Infof(ctx, "Getting chat model, model ID: %s", chatManage.ChatModelID)
chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return nil, nil, err
}
logger.Info(ctx, "Setting up chat options")
opt := &chat.ChatOptions{
Temperature: chatManage.SummaryConfig.Temperature,
TopP: chatManage.SummaryConfig.TopP,
@@ -94,6 +94,7 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
return next()
}
// Extractor is a struct for extracting entities
type Extractor struct {
chat chat.Chat
formater *Formater
@@ -101,6 +102,7 @@ type Extractor struct {
chatOpt *chat.ChatOptions
}
// NewExtractor creates a new extractor
func NewExtractor(
chatModel chat.Chat,
template *types.PromptTemplateStructured,
@@ -118,6 +120,7 @@ func NewExtractor(
}
}
// Extract extracts entities from content
func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) {
generator := NewQAPromptGenerator(e.formater, e.template)
@@ -139,6 +142,7 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa
return graph, nil
}
// RemoveUnknownRelation removes unknown relations from graph
func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) {
relationType := make(map[string]bool)
for _, tag := range e.template.Tags {
@@ -156,6 +160,7 @@ func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.Grap
graph.Relation = relationNew
}
// QAPromptGenerator is a struct for generating QA prompts
type QAPromptGenerator struct {
Formater *Formater
Template *types.PromptTemplateStructured
@@ -165,6 +170,7 @@ type QAPromptGenerator struct {
AnswerPrefix string
}
// NewQAPromptGenerator creates a new QA prompt generator
func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator {
return &QAPromptGenerator{
Formater: formater,
@@ -176,6 +182,7 @@ func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStru
}
}
// System generates a system prompt
func (qa *QAPromptGenerator) System(ctx context.Context) string {
promptLines := []string{}
@@ -205,6 +212,7 @@ func (qa *QAPromptGenerator) System(ctx context.Context) string {
return strings.Join(promptLines, "\n")
}
// User generates a user prompt
func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
promptLines := []string{}
promptLines = append(promptLines, qa.QuestionHeading)
@@ -213,6 +221,7 @@ func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
return strings.Join(promptLines, "\n")
}
// Render renders a prompt
func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message {
return []chat.Message{
{
@@ -226,10 +235,13 @@ func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat
}
}
// FormatType is a type for format types
type FormatType string
const (
// FormatTypeJSON is a format type for JSON
FormatTypeJSON FormatType = "json"
// FormatTypeYAML is a format type for YAML
FormatTypeYAML FormatType = "yaml"
)
@@ -245,6 +257,7 @@ var _FENCE_RE = regexp.MustCompile(
_FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END,
)
// Formater is a struct for formatting entities
type Formater struct {
attributeSuffix string
formatType FormatType
@@ -256,6 +269,7 @@ type Formater struct {
relationPrefix string
}
// NewFormater creates a new formater
func NewFormater() *Formater {
return &Formater{
attributeSuffix: "_attributes",
@@ -268,6 +282,7 @@ func NewFormater() *Formater {
}
}
// formatExtraction formats extraction
func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) {
items := make([]map[string]interface{}, 0)
for _, node := range nodes {
@@ -51,6 +51,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
"chunk_cnt": len(searchResult),
})
// If there are no search results, return early
if len(searchResult) == 0 {
pipelineWarn(ctx, "Merge", "output", map[string]interface{}{
"chunk_cnt": 0,
@@ -212,6 +213,7 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
return nil
}
// populateFAQAnswers populates FAQ answers for the search results
func (p *PluginMerge) populateFAQAnswers(
ctx context.Context,
chatManage *types.ChatManage,
@@ -300,6 +302,7 @@ func (p *PluginMerge) populateFAQAnswers(
return results
}
// buildFAQAnswerContent builds the content of a FAQ answer
func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
if meta == nil {
return ""
@@ -336,6 +339,7 @@ func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
return strings.TrimSpace(builder.String())
}
// expandShortContextWithNeighbors expands the short context with neighbors
func (p *PluginMerge) expandShortContextWithNeighbors(
ctx context.Context,
chatManage *types.ChatManage,
@@ -580,10 +584,12 @@ func (p *PluginMerge) expandShortContextWithNeighbors(
return results
}
// runeLen returns the length of a string in runes
func runeLen(s string) int {
return len([]rune(s))
}
// mergeOrderedContent merges ordered content
func mergeOrderedContent(prev, base, next string, maxLen int) string {
content := base
if prev != "" {
@@ -599,7 +605,7 @@ func mergeOrderedContent(prev, base, next string, maxLen int) string {
return content
}
// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix
// concatNoOverlap concatenates two strings, removing potential overlapping prefix/suffix
func concatNoOverlap(a, b string) string {
if a == "" {
return b
@@ -239,6 +239,7 @@ func isInformativeToken(token string) bool {
return true
}
// containsChineseCharacters checks if a string contains Chinese characters
func containsChineseCharacters(text string) bool {
for _, r := range text {
if unicode.Is(unicode.Han, r) {
@@ -248,6 +249,7 @@ func containsChineseCharacters(text string) bool {
return false
}
// containsLatinLetters checks if a string contains Latin letters
func containsLatinLetters(text string) bool {
for _, r := range text {
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
@@ -257,22 +259,26 @@ func containsLatinLetters(text string) bool {
return false
}
// normalizeWhitespace normalizes whitespace in a string
func normalizeWhitespace(text string) string {
text = strings.TrimSpace(text)
return multiSpaceRegex.ReplaceAllString(text, " ")
}
// normalizeLatinQuery normalizes a Latin query
func normalizeLatinQuery(text string) string {
text = strings.ToLower(text)
text = multiSpaceRegex.ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}
// intentResp is a response for intent detection
type intentResp struct {
Intent string `json:"intent"`
Confidence float64 `json:"confidence"`
}
// detectIntentLLM detects the intent of a query using an LLM
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
if p.modelService == nil || chatManage.ChatModelID == "" {
pipelineWarn(
@@ -330,6 +336,7 @@ func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *type
}
}
// extractJSONBody extracts a JSON body from a string
func extractJSONBody(text string) string {
t := strings.TrimSpace(text)
// Try fenced block first
@@ -176,6 +176,7 @@ func (p *PluginRerank) rerank(ctx context.Context,
return rankFilter
}
// ensureMetadata ensures the metadata is not nil
func ensureMetadata(m map[string]string) map[string]string {
if m == nil {
return make(map[string]string)
@@ -183,6 +184,7 @@ func ensureMetadata(m map[string]string) map[string]string {
return m
}
// compositeScore calculates the composite score for a search result
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
sourceWeight := 1.0
switch strings.ToLower(sr.KnowledgeSource) {
@@ -222,6 +224,7 @@ func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatM
return composite
}
// applyMMR applies the MMR algorithm to the search results
func applyMMR(
ctx context.Context,
results []*types.SearchResult,
@@ -285,6 +288,7 @@ func applyMMR(
return selected
}
// tokenizeSimple tokenizes a text into a set of tokens
func tokenizeSimple(text string) map[string]struct{} {
text = strings.ToLower(text)
fields := strings.Fields(text)
@@ -297,6 +301,7 @@ func tokenizeSimple(text string) map[string]struct{} {
return set
}
// jaccard calculates the Jaccard similarity between two sets of tokens
func jaccard(a, b map[string]struct{}) float64 {
if len(a) == 0 && len(b) == 0 {
return 0
@@ -314,6 +319,7 @@ func jaccard(a, b map[string]struct{}) float64 {
return float64(inter) / float64(union)
}
// clampFloat clamps a float value between a minimum and maximum value
func clampFloat(v, minV, maxV float64) float64 {
if v < minV {
return minV
@@ -16,6 +16,7 @@ type PluginSearchEntity struct {
knowledgeRepo interfaces.KnowledgeRepository
}
// NewPluginSearchEntity creates a new plugin search entity
func NewPluginSearchEntity(
eventManager *EventManager,
graphRepository interfaces.RetrieveGraphRepository,
@@ -31,12 +32,12 @@ func NewPluginSearchEntity(
return res
}
// ActivationEvents returns the event types this plugin handles
// ActivationEvents returns the list of event types this plugin responds to
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
return []types.EventType{types.ENTITY_SEARCH}
}
// OnEvent handles search events in the chat pipeline
// OnEvent processes triggered events
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
@@ -148,6 +149,7 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
return next()
}
// filterSeenChunk filters seen chunks from the graph
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
seen := map[string]bool{}
for _, chunk := range searchResult {
@@ -169,6 +171,7 @@ func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult [
return chunkIDs
}
// chunk2SearchResult converts a chunk to a search result
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
return &types.SearchResult{
ID: chunk.ID,
+4 -2
View File
@@ -16,6 +16,7 @@ import (
"github.com/hibiken/asynq"
)
// NewChunkExtractTask creates a new chunk extract task
func NewChunkExtractTask(
ctx context.Context,
client *asynq.Client,
@@ -45,6 +46,7 @@ func NewChunkExtractTask(
return nil
}
// ChunkExtractService is a service for extracting chunks
type ChunkExtractService struct {
template *types.PromptTemplateStructured
modelService interfaces.ModelService
@@ -53,6 +55,7 @@ type ChunkExtractService struct {
graphEngine interfaces.RetrieveGraphRepository
}
// NewChunkExtractService creates a new chunk extract service
func NewChunkExtractService(
config *config.Config,
modelService interfaces.ModelService,
@@ -73,6 +76,7 @@ func NewChunkExtractService(
}
}
// Extract extracts a chunk
func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error {
var p types.ExtractChunkPayload
if err := json.Unmarshal(t.Payload(), &p); err != nil {
@@ -137,7 +141,5 @@ func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error
logger.Errorf(ctx, "failed to add graph: %v", err)
return err
}
// gg, _ := json.Marshal(graph)
// logger.Infof(ctx, "extracted graph: %s", string(gg))
return nil
}
+2 -2
View File
@@ -193,12 +193,12 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
logger.GetLogger(ctx).Infof("MCP service disabled, connection closed: %s (ID: %s)", name, service.ID)
} else if configChanged {
s.mcpManager.CloseClient(service.ID)
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed for reconnect: %s (ID: %s)", name, service.ID)
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed: %s (ID: %s)", name, service.ID)
} else if oldEnabled != existing.Enabled && existing.Enabled {
// Service was just enabled (was disabled, now enabled)
// Close any existing connection to ensure clean state
s.mcpManager.CloseClient(service.ID)
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed for clean state: %s (ID: %s)", name, service.ID)
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed: %s (ID: %s)", name, service.ID)
}
logger.GetLogger(ctx).Infof("MCP service updated: %s (ID: %s), enabled: %v", name, service.ID, existing.Enabled)
+6 -3
View File
@@ -634,7 +634,8 @@ func (s *sessionService) selectChatModelID(
return session.SummaryModelID, nil
} else if err == nil && model != nil {
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
logger.Infof(ctx, "Session has summary model %s but it's not Remote, checking knowledge bases for Remote models", session.SummaryModelID)
logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+
"checking knowledge bases for Remote models", session.SummaryModelID)
}
}
@@ -952,7 +953,8 @@ func (s *sessionService) AgentQA(
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
}
} else {
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v", len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
}
summaryModelID := session.SummaryModelID
@@ -1141,7 +1143,8 @@ func (s *sessionService) SaveWebSearchTempKBState(
}
}
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis
// and cleans up associated knowledge base and knowledge items.
func (s *sessionService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error {
if s.redisClient == nil {
return nil
@@ -63,6 +63,7 @@ func (p *DuckDuckGoProvider) Search(
return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr)
}
// searchHTML performs a web search using DuckDuckGo HTML endpoint
func (p *DuckDuckGoProvider) searchHTML(
ctx context.Context,
query string,
@@ -134,6 +135,7 @@ func (p *DuckDuckGoProvider) searchHTML(
return results, nil
}
// searchAPI performs a web search using DuckDuckGo API endpoint
func (p *DuckDuckGoProvider) searchAPI(
ctx context.Context,
query string,
@@ -220,6 +222,7 @@ func (p *DuckDuckGoProvider) searchAPI(
return results, nil
}
// cleanDDGURL cleans the URL from DuckDuckGo HTML endpoint
func cleanDDGURL(urlStr string) string {
if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") {
trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=")
@@ -241,6 +244,7 @@ func cleanDDGURL(urlStr string) string {
return urlStr
}
// extractTitle extracts the title from the text
func extractTitle(text string) string {
lines := strings.Split(text, "\n")
if len(lines) > 0 {
@@ -52,7 +52,7 @@ func TestDuckDuckGoProvider_Name(t *testing.T) {
}
}
func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
func TestDuckDuckGoProvider(t *testing.T) {
// Minimal HTML page with two results, matching selectors used in searchHTML
html := `
<html>
@@ -105,7 +105,7 @@ func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
}
}
func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) {
func TestDuckDuckGoProvider_Fallback(t *testing.T) {
// Simulate HTML returning non-OK to force API fallback, then a minimal API JSON
apiResp := struct {
AbstractText string `json:"AbstractText"`
@@ -233,7 +233,7 @@ func TestDuckDuckGoProvider_Search_Real(t *testing.T) {
// TestDuckDuckGoProvider_Search_Real_Chinese tests the DuckDuckGo provider with Chinese query.
// This verifies the Chinese language parameter (kl=cn-zh) works correctly.
func TestDuckDuckGoProvider_Search_Real_Chinese(t *testing.T) {
func TestDuckDuckGoProvider_SearchChinese(t *testing.T) {
if testing.Short() {
t.Skip("Skipping real DuckDuckGo integration test in short mode")
}