refactor: Remove agent_test command-line tool and associated files to streamline project structure and eliminate unused components
This commit is contained in:
@@ -11,19 +11,23 @@ import (
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
)
|
||||
|
||||
// Neo4jRepository is a repository for Neo4j
|
||||
type Neo4jRepository struct {
|
||||
driver neo4j.Driver
|
||||
nodePrefix string
|
||||
}
|
||||
|
||||
// NewNeo4jRepository creates a new Neo4j repository
|
||||
func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository {
|
||||
return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"}
|
||||
}
|
||||
|
||||
// _remove_hyphen removes hyphens from a string
|
||||
func _remove_hyphen(s string) string {
|
||||
return strings.ReplaceAll(s, "-", "_")
|
||||
}
|
||||
|
||||
// Labels returns the labels for a namespace
|
||||
func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
|
||||
res := make([]string, 0)
|
||||
for _, label := range namespace.Labels() {
|
||||
@@ -32,12 +36,13 @@ func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
|
||||
return res
|
||||
}
|
||||
|
||||
// Label returns the label for a namespace
|
||||
func (n *Neo4jRepository) Label(namespace types.NameSpace) string {
|
||||
labels := n.Labels(namespace)
|
||||
return strings.Join(labels, ":")
|
||||
}
|
||||
|
||||
// AddGraph implements interfaces.RetrieveGraphRepository.
|
||||
// AddGraph adds a graph to the Neo4j repository
|
||||
func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
@@ -51,11 +56,13 @@ func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil
|
||||
}
|
||||
|
||||
// addGraph adds a graph to the Neo4j repository
|
||||
func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error {
|
||||
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
|
||||
defer session.Close(ctx)
|
||||
|
||||
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
|
||||
// Node import query
|
||||
node_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node
|
||||
@@ -76,6 +83,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil, fmt.Errorf("failed to create nodes: %v", err)
|
||||
}
|
||||
|
||||
// Relationship import query
|
||||
rel_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source
|
||||
@@ -106,7 +114,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil
|
||||
}
|
||||
|
||||
// DelGraph implements interfaces.RetrieveGraphRepository.
|
||||
// DelGraph deletes a graph from the Neo4j repository
|
||||
func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
@@ -152,6 +160,7 @@ func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameS
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchNode searches for nodes in the Neo4j repository
|
||||
func (n *Neo4jRepository) SearchNode(
|
||||
ctx context.Context,
|
||||
namespace types.NameSpace,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// knowledgeTagRepository is a repository for knowledge tags
|
||||
type knowledgeTagRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
@@ -17,14 +18,17 @@ func NewKnowledgeTagRepository(db *gorm.DB) interfaces.KnowledgeTagRepository {
|
||||
return &knowledgeTagRepository{db: db}
|
||||
}
|
||||
|
||||
// Create creates a new knowledge tag
|
||||
func (r *knowledgeTagRepository) Create(ctx context.Context, tag *types.KnowledgeTag) error {
|
||||
return r.db.WithContext(ctx).Create(tag).Error
|
||||
}
|
||||
|
||||
// Update updates a knowledge tag
|
||||
func (r *knowledgeTagRepository) Update(ctx context.Context, tag *types.KnowledgeTag) error {
|
||||
return r.db.WithContext(ctx).Save(tag).Error
|
||||
}
|
||||
|
||||
// GetByID gets a knowledge tag by ID
|
||||
func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.KnowledgeTag, error) {
|
||||
var tag types.KnowledgeTag
|
||||
if err := r.db.WithContext(ctx).
|
||||
@@ -35,6 +39,7 @@ func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, i
|
||||
return &tag, nil
|
||||
}
|
||||
|
||||
// ListByKB lists knowledge tags by knowledge base ID
|
||||
func (r *knowledgeTagRepository) ListByKB(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
@@ -50,13 +55,14 @@ func (r *knowledgeTagRepository) ListByKB(
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// Delete deletes a knowledge tag
|
||||
func (r *knowledgeTagRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
|
||||
return r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND id = ?", tenantID, id).
|
||||
Delete(&types.KnowledgeTag{}).Error
|
||||
}
|
||||
|
||||
// CountReferences returns how many knowledges and chunks reference this tag.
|
||||
// CountReferences returns the number of knowledges and chunks that reference this tag
|
||||
func (r *knowledgeTagRepository) CountReferences(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
|
||||
@@ -10,31 +10,32 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// pipelineInfo logs pipeline info level entries.
|
||||
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineInfo(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// pipelineWarn logs pipeline warning level entries.
|
||||
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineWarn(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// pipelineError logs pipeline error level entries.
|
||||
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineError(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// prepareChatModel shared logic to prepare chat model and options
|
||||
// it gets the chat model and sets up the chat options based on the chat manage.
|
||||
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
|
||||
chatManage *types.ChatManage,
|
||||
) (chat.Chat, *chat.ChatOptions, error) {
|
||||
logger.Infof(ctx, "Getting chat model, model ID: %s", chatManage.ChatModelID)
|
||||
|
||||
chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chat model: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Setting up chat options")
|
||||
opt := &chat.ChatOptions{
|
||||
Temperature: chatManage.SummaryConfig.Temperature,
|
||||
TopP: chatManage.SummaryConfig.TopP,
|
||||
|
||||
@@ -94,6 +94,7 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// Extractor is a struct for extracting entities
|
||||
type Extractor struct {
|
||||
chat chat.Chat
|
||||
formater *Formater
|
||||
@@ -101,6 +102,7 @@ type Extractor struct {
|
||||
chatOpt *chat.ChatOptions
|
||||
}
|
||||
|
||||
// NewExtractor creates a new extractor
|
||||
func NewExtractor(
|
||||
chatModel chat.Chat,
|
||||
template *types.PromptTemplateStructured,
|
||||
@@ -118,6 +120,7 @@ func NewExtractor(
|
||||
}
|
||||
}
|
||||
|
||||
// Extract extracts entities from content
|
||||
func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) {
|
||||
generator := NewQAPromptGenerator(e.formater, e.template)
|
||||
|
||||
@@ -139,6 +142,7 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// RemoveUnknownRelation removes unknown relations from graph
|
||||
func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) {
|
||||
relationType := make(map[string]bool)
|
||||
for _, tag := range e.template.Tags {
|
||||
@@ -156,6 +160,7 @@ func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.Grap
|
||||
graph.Relation = relationNew
|
||||
}
|
||||
|
||||
// QAPromptGenerator is a struct for generating QA prompts
|
||||
type QAPromptGenerator struct {
|
||||
Formater *Formater
|
||||
Template *types.PromptTemplateStructured
|
||||
@@ -165,6 +170,7 @@ type QAPromptGenerator struct {
|
||||
AnswerPrefix string
|
||||
}
|
||||
|
||||
// NewQAPromptGenerator creates a new QA prompt generator
|
||||
func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator {
|
||||
return &QAPromptGenerator{
|
||||
Formater: formater,
|
||||
@@ -176,6 +182,7 @@ func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStru
|
||||
}
|
||||
}
|
||||
|
||||
// System generates a system prompt
|
||||
func (qa *QAPromptGenerator) System(ctx context.Context) string {
|
||||
promptLines := []string{}
|
||||
|
||||
@@ -205,6 +212,7 @@ func (qa *QAPromptGenerator) System(ctx context.Context) string {
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
// User generates a user prompt
|
||||
func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
|
||||
promptLines := []string{}
|
||||
promptLines = append(promptLines, qa.QuestionHeading)
|
||||
@@ -213,6 +221,7 @@ func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
// Render renders a prompt
|
||||
func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message {
|
||||
return []chat.Message{
|
||||
{
|
||||
@@ -226,10 +235,13 @@ func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat
|
||||
}
|
||||
}
|
||||
|
||||
// FormatType is a type for format types
|
||||
type FormatType string
|
||||
|
||||
const (
|
||||
// FormatTypeJSON is a format type for JSON
|
||||
FormatTypeJSON FormatType = "json"
|
||||
// FormatTypeYAML is a format type for YAML
|
||||
FormatTypeYAML FormatType = "yaml"
|
||||
)
|
||||
|
||||
@@ -245,6 +257,7 @@ var _FENCE_RE = regexp.MustCompile(
|
||||
_FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END,
|
||||
)
|
||||
|
||||
// Formater is a struct for formatting entities
|
||||
type Formater struct {
|
||||
attributeSuffix string
|
||||
formatType FormatType
|
||||
@@ -256,6 +269,7 @@ type Formater struct {
|
||||
relationPrefix string
|
||||
}
|
||||
|
||||
// NewFormater creates a new formater
|
||||
func NewFormater() *Formater {
|
||||
return &Formater{
|
||||
attributeSuffix: "_attributes",
|
||||
@@ -268,6 +282,7 @@ func NewFormater() *Formater {
|
||||
}
|
||||
}
|
||||
|
||||
// formatExtraction formats extraction
|
||||
func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) {
|
||||
items := make([]map[string]interface{}, 0)
|
||||
for _, node := range nodes {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
"chunk_cnt": len(searchResult),
|
||||
})
|
||||
|
||||
// If there are no search results, return early
|
||||
if len(searchResult) == 0 {
|
||||
pipelineWarn(ctx, "Merge", "output", map[string]interface{}{
|
||||
"chunk_cnt": 0,
|
||||
@@ -212,6 +213,7 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
return nil
|
||||
}
|
||||
|
||||
// populateFAQAnswers populates FAQ answers for the search results
|
||||
func (p *PluginMerge) populateFAQAnswers(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
@@ -300,6 +302,7 @@ func (p *PluginMerge) populateFAQAnswers(
|
||||
return results
|
||||
}
|
||||
|
||||
// buildFAQAnswerContent builds the content of a FAQ answer
|
||||
func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
|
||||
if meta == nil {
|
||||
return ""
|
||||
@@ -336,6 +339,7 @@ func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
|
||||
return strings.TrimSpace(builder.String())
|
||||
}
|
||||
|
||||
// expandShortContextWithNeighbors expands the short context with neighbors
|
||||
func (p *PluginMerge) expandShortContextWithNeighbors(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
@@ -580,10 +584,12 @@ func (p *PluginMerge) expandShortContextWithNeighbors(
|
||||
return results
|
||||
}
|
||||
|
||||
// runeLen returns the length of a string in runes
|
||||
func runeLen(s string) int {
|
||||
return len([]rune(s))
|
||||
}
|
||||
|
||||
// mergeOrderedContent merges ordered content
|
||||
func mergeOrderedContent(prev, base, next string, maxLen int) string {
|
||||
content := base
|
||||
if prev != "" {
|
||||
@@ -599,7 +605,7 @@ func mergeOrderedContent(prev, base, next string, maxLen int) string {
|
||||
return content
|
||||
}
|
||||
|
||||
// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix
|
||||
// concatNoOverlap concatenates two strings, removing potential overlapping prefix/suffix
|
||||
func concatNoOverlap(a, b string) string {
|
||||
if a == "" {
|
||||
return b
|
||||
|
||||
@@ -239,6 +239,7 @@ func isInformativeToken(token string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// containsChineseCharacters checks if a string contains Chinese characters
|
||||
func containsChineseCharacters(text string) bool {
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
@@ -248,6 +249,7 @@ func containsChineseCharacters(text string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// containsLatinLetters checks if a string contains Latin letters
|
||||
func containsLatinLetters(text string) bool {
|
||||
for _, r := range text {
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
@@ -257,22 +259,26 @@ func containsLatinLetters(text string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// normalizeWhitespace normalizes whitespace in a string
|
||||
func normalizeWhitespace(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
return multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
}
|
||||
|
||||
// normalizeLatinQuery normalizes a Latin query
|
||||
func normalizeLatinQuery(text string) string {
|
||||
text = strings.ToLower(text)
|
||||
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// intentResp is a response for intent detection
|
||||
type intentResp struct {
|
||||
Intent string `json:"intent"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// detectIntentLLM detects the intent of a query using an LLM
|
||||
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
|
||||
if p.modelService == nil || chatManage.ChatModelID == "" {
|
||||
pipelineWarn(
|
||||
@@ -330,6 +336,7 @@ func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *type
|
||||
}
|
||||
}
|
||||
|
||||
// extractJSONBody extracts a JSON body from a string
|
||||
func extractJSONBody(text string) string {
|
||||
t := strings.TrimSpace(text)
|
||||
// Try fenced block first
|
||||
|
||||
@@ -176,6 +176,7 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
||||
return rankFilter
|
||||
}
|
||||
|
||||
// ensureMetadata ensures the metadata is not nil
|
||||
func ensureMetadata(m map[string]string) map[string]string {
|
||||
if m == nil {
|
||||
return make(map[string]string)
|
||||
@@ -183,6 +184,7 @@ func ensureMetadata(m map[string]string) map[string]string {
|
||||
return m
|
||||
}
|
||||
|
||||
// compositeScore calculates the composite score for a search result
|
||||
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
|
||||
sourceWeight := 1.0
|
||||
switch strings.ToLower(sr.KnowledgeSource) {
|
||||
@@ -222,6 +224,7 @@ func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatM
|
||||
return composite
|
||||
}
|
||||
|
||||
// applyMMR applies the MMR algorithm to the search results
|
||||
func applyMMR(
|
||||
ctx context.Context,
|
||||
results []*types.SearchResult,
|
||||
@@ -285,6 +288,7 @@ func applyMMR(
|
||||
return selected
|
||||
}
|
||||
|
||||
// tokenizeSimple tokenizes a text into a set of tokens
|
||||
func tokenizeSimple(text string) map[string]struct{} {
|
||||
text = strings.ToLower(text)
|
||||
fields := strings.Fields(text)
|
||||
@@ -297,6 +301,7 @@ func tokenizeSimple(text string) map[string]struct{} {
|
||||
return set
|
||||
}
|
||||
|
||||
// jaccard calculates the Jaccard similarity between two sets of tokens
|
||||
func jaccard(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 0
|
||||
@@ -314,6 +319,7 @@ func jaccard(a, b map[string]struct{}) float64 {
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
|
||||
// clampFloat clamps a float value between a minimum and maximum value
|
||||
func clampFloat(v, minV, maxV float64) float64 {
|
||||
if v < minV {
|
||||
return minV
|
||||
|
||||
@@ -16,6 +16,7 @@ type PluginSearchEntity struct {
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
}
|
||||
|
||||
// NewPluginSearchEntity creates a new plugin search entity
|
||||
func NewPluginSearchEntity(
|
||||
eventManager *EventManager,
|
||||
graphRepository interfaces.RetrieveGraphRepository,
|
||||
@@ -31,12 +32,12 @@ func NewPluginSearchEntity(
|
||||
return res
|
||||
}
|
||||
|
||||
// ActivationEvents returns the event types this plugin handles
|
||||
// ActivationEvents returns the list of event types this plugin responds to
|
||||
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.ENTITY_SEARCH}
|
||||
}
|
||||
|
||||
// OnEvent handles search events in the chat pipeline
|
||||
// OnEvent processes triggered events
|
||||
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
@@ -148,6 +149,7 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// filterSeenChunk filters seen chunks from the graph
|
||||
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, chunk := range searchResult {
|
||||
@@ -169,6 +171,7 @@ func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult [
|
||||
return chunkIDs
|
||||
}
|
||||
|
||||
// chunk2SearchResult converts a chunk to a search result
|
||||
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
|
||||
return &types.SearchResult{
|
||||
ID: chunk.ID,
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// NewChunkExtractTask creates a new chunk extract task
|
||||
func NewChunkExtractTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
@@ -45,6 +46,7 @@ func NewChunkExtractTask(
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChunkExtractService is a service for extracting chunks
|
||||
type ChunkExtractService struct {
|
||||
template *types.PromptTemplateStructured
|
||||
modelService interfaces.ModelService
|
||||
@@ -53,6 +55,7 @@ type ChunkExtractService struct {
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
}
|
||||
|
||||
// NewChunkExtractService creates a new chunk extract service
|
||||
func NewChunkExtractService(
|
||||
config *config.Config,
|
||||
modelService interfaces.ModelService,
|
||||
@@ -73,6 +76,7 @@ func NewChunkExtractService(
|
||||
}
|
||||
}
|
||||
|
||||
// Extract extracts a chunk
|
||||
func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error {
|
||||
var p types.ExtractChunkPayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
@@ -137,7 +141,5 @@ func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error
|
||||
logger.Errorf(ctx, "failed to add graph: %v", err)
|
||||
return err
|
||||
}
|
||||
// gg, _ := json.Marshal(graph)
|
||||
// logger.Infof(ctx, "extracted graph: %s", string(gg))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -193,12 +193,12 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
|
||||
logger.GetLogger(ctx).Infof("MCP service disabled, connection closed: %s (ID: %s)", name, service.ID)
|
||||
} else if configChanged {
|
||||
s.mcpManager.CloseClient(service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed for reconnect: %s (ID: %s)", name, service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed: %s (ID: %s)", name, service.ID)
|
||||
} else if oldEnabled != existing.Enabled && existing.Enabled {
|
||||
// Service was just enabled (was disabled, now enabled)
|
||||
// Close any existing connection to ensure clean state
|
||||
s.mcpManager.CloseClient(service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed for clean state: %s (ID: %s)", name, service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed: %s (ID: %s)", name, service.ID)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("MCP service updated: %s (ID: %s), enabled: %v", name, service.ID, existing.Enabled)
|
||||
|
||||
@@ -634,7 +634,8 @@ func (s *sessionService) selectChatModelID(
|
||||
return session.SummaryModelID, nil
|
||||
} else if err == nil && model != nil {
|
||||
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+
|
||||
"checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -952,7 +953,8 @@ func (s *sessionService) AgentQA(
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
}
|
||||
} else {
|
||||
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v", len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
}
|
||||
|
||||
summaryModelID := session.SummaryModelID
|
||||
@@ -1141,7 +1143,8 @@ func (s *sessionService) SaveWebSearchTempKBState(
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis
|
||||
// and cleans up associated knowledge base and knowledge items.
|
||||
func (s *sessionService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error {
|
||||
if s.redisClient == nil {
|
||||
return nil
|
||||
|
||||
@@ -63,6 +63,7 @@ func (p *DuckDuckGoProvider) Search(
|
||||
return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr)
|
||||
}
|
||||
|
||||
// searchHTML performs a web search using DuckDuckGo HTML endpoint
|
||||
func (p *DuckDuckGoProvider) searchHTML(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
@@ -134,6 +135,7 @@ func (p *DuckDuckGoProvider) searchHTML(
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// searchAPI performs a web search using DuckDuckGo API endpoint
|
||||
func (p *DuckDuckGoProvider) searchAPI(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
@@ -220,6 +222,7 @@ func (p *DuckDuckGoProvider) searchAPI(
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// cleanDDGURL cleans the URL from DuckDuckGo HTML endpoint
|
||||
func cleanDDGURL(urlStr string) string {
|
||||
if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") {
|
||||
trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=")
|
||||
@@ -241,6 +244,7 @@ func cleanDDGURL(urlStr string) string {
|
||||
return urlStr
|
||||
}
|
||||
|
||||
// extractTitle extracts the title from the text
|
||||
func extractTitle(text string) string {
|
||||
lines := strings.Split(text, "\n")
|
||||
if len(lines) > 0 {
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestDuckDuckGoProvider_Name(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
func TestDuckDuckGoProvider(t *testing.T) {
|
||||
// Minimal HTML page with two results, matching selectors used in searchHTML
|
||||
html := `
|
||||
<html>
|
||||
@@ -105,7 +105,7 @@ func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) {
|
||||
func TestDuckDuckGoProvider_Fallback(t *testing.T) {
|
||||
// Simulate HTML returning non-OK to force API fallback, then a minimal API JSON
|
||||
apiResp := struct {
|
||||
AbstractText string `json:"AbstractText"`
|
||||
@@ -233,7 +233,7 @@ func TestDuckDuckGoProvider_Search_Real(t *testing.T) {
|
||||
|
||||
// TestDuckDuckGoProvider_Search_Real_Chinese tests the DuckDuckGo provider with Chinese query.
|
||||
// This verifies the Chinese language parameter (kl=cn-zh) works correctly.
|
||||
func TestDuckDuckGoProvider_Search_Real_Chinese(t *testing.T) {
|
||||
func TestDuckDuckGoProvider_SearchChinese(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping real DuckDuckGo integration test in short mode")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user