wizardchen
057cfd9eb0
- Added support for tracking the status of FAQ import tasks, including progress indicators for ongoing imports. - Introduced new localization strings for processing tasks and batch operations in multiple languages. - Updated the FAQ entry manager to display import progress and improved user feedback during the import process. - Enhanced the KnowledgeBase component to reflect the processing state of knowledge items, improving user experience during document uploads. - Implemented batch deletion functionality for chunks, streamlining the management of knowledge base content.
480 lines
15 KiB
Go
480 lines
15 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// GrepChunksTool performs text pattern matching in knowledge base chunks
|
|
// Similar to grep command in Unix-like systems, but operates on knowledge base content
|
|
type GrepChunksTool struct {
|
|
BaseTool
|
|
db *gorm.DB
|
|
tenantID uint
|
|
knowledgeBaseIDs []string
|
|
}
|
|
|
|
// NewGrepChunksTool creates a new grep chunks tool
|
|
func NewGrepChunksTool(db *gorm.DB, tenantID uint, knowledgeBaseIDs []string) *GrepChunksTool {
|
|
description := `Unix-style text pattern matching tool for knowledge base chunks.
|
|
|
|
Searches for text patterns in chunk content, similar to the Unix grep command.
|
|
|
|
## Core Function
|
|
Performs **exact text pattern matching** (NOT semantic search). Finds chunks containing any of the specified patterns using literal text matching (fixed string). Supports multiple patterns with OR logic.
|
|
|
|
## CRITICAL - Keyword Granularity
|
|
**MUST use SHORT keywords (1-3 words), NOT long phrases.** Break down long phrases into smaller keywords for better match rate.
|
|
|
|
**Why**: Long phrases like "中国饮食文化" may not match if the document contains "中国" and "饮食" and "文化" separately but not the exact phrase.
|
|
|
|
**Guidelines**:
|
|
- ❌ **Bad**: ["中国饮食文化", "日本饮食文化"] - too long, may not match
|
|
- ✅ **Good**: ["中国", "饮食", "文化", "日本", "料理", "和食"] - short keywords, higher match rate
|
|
- For comparisons: Extract keywords for each entity separately
|
|
- Use single words or 2-word phrases, avoid 3+ word phrases
|
|
|
|
**Examples**:
|
|
- "中国饮食文化" → ["中国", "饮食", "文化", "中餐", "中华"]
|
|
- "日本饮食文化" → ["日本", "饮食", "文化", "料理", "和食", "日式"]
|
|
- "对比中国和日本" → Search separately: ["中国", "中华"] then ["日本", "日式"]
|
|
|
|
## Usage
|
|
grep_chunks searches through all enabled chunks in the knowledge base(s) and displays matching chunks with context. When multiple patterns are provided, results match any pattern (OR logic).
|
|
|
|
Default behavior: case-insensitive matching, shows chunk indices, displays context around matches.
|
|
|
|
## When to Use
|
|
- Finding specific entities: "FAISS", "Redis", "404", "RAG"
|
|
- Exact keyword lookup
|
|
- Quick text search before semantic search
|
|
|
|
## Examples
|
|
- Single pattern: pattern=["FAISS"]
|
|
- Multiple patterns: pattern=["向量", "vector", "embedding"]
|
|
- Search synonyms: pattern=["RAG", "检索增强生成"]
|
|
- Short keywords: pattern=["中国", "饮食", "文化"] (NOT ["中国饮食文化"])
|
|
`
|
|
|
|
return &GrepChunksTool{
|
|
BaseTool: NewBaseTool("grep_chunks", description),
|
|
db: db,
|
|
tenantID: tenantID,
|
|
knowledgeBaseIDs: knowledgeBaseIDs,
|
|
}
|
|
}
|
|
|
|
// Parameters returns the JSON schema for the tool's parameters
|
|
func (t *GrepChunksTool) Parameters() map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"pattern": map[string]interface{}{
|
|
"type": "array",
|
|
"description": "REQUIRED: Text patterns to search for. Can be a single pattern or multiple patterns. Treated as literal text (fixed string matching). Results match any of the patterns (OR logic).",
|
|
"items": map[string]interface{}{
|
|
"type": "string",
|
|
},
|
|
"minItems": 1,
|
|
},
|
|
"knowledge_base_ids": map[string]interface{}{
|
|
"type": "array",
|
|
"description": "Filter by knowledge base IDs. If empty, searches all allowed KBs.",
|
|
"items": map[string]interface{}{
|
|
"type": "string",
|
|
},
|
|
},
|
|
"knowledge_ids": map[string]interface{}{
|
|
"type": "array",
|
|
"description": "Filter by document/knowledge IDs. If empty, searches all documents.",
|
|
"items": map[string]interface{}{
|
|
"type": "string",
|
|
},
|
|
},
|
|
"max_results": map[string]interface{}{
|
|
"type": "integer",
|
|
"description": "Maximum number of matching chunks to return (default: 50, max: 200)",
|
|
"default": 50,
|
|
"minimum": 1,
|
|
"maximum": 200,
|
|
},
|
|
},
|
|
"required": []string{"pattern"},
|
|
}
|
|
}
|
|
|
|
// Execute executes the grep chunks tool
|
|
func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
|
|
logger.Infof(ctx, "[Tool][GrepChunks] Execute started")
|
|
|
|
// Parse pattern parameter (required) - support multiple patterns
|
|
var patterns []string
|
|
if patternsRaw, ok := args["pattern"].([]interface{}); ok && len(patternsRaw) > 0 {
|
|
for _, p := range patternsRaw {
|
|
if pStr, ok := p.(string); ok && strings.TrimSpace(pStr) != "" {
|
|
patterns = append(patterns, strings.TrimSpace(pStr))
|
|
}
|
|
}
|
|
}
|
|
// Also support single string for backward compatibility
|
|
if len(patterns) == 0 {
|
|
if patternStr, ok := args["pattern"].(string); ok && strings.TrimSpace(patternStr) != "" {
|
|
patterns = append(patterns, strings.TrimSpace(patternStr))
|
|
}
|
|
}
|
|
if len(patterns) == 0 {
|
|
logger.Errorf(ctx, "[Tool][GrepChunks] Missing or invalid pattern parameter")
|
|
return &types.ToolResult{
|
|
Success: false,
|
|
Error: "pattern parameter is required and must contain at least one non-empty pattern",
|
|
}, fmt.Errorf("missing pattern parameter")
|
|
}
|
|
|
|
// Use default values for all options
|
|
contextLines := 50 // default: 50 context characters
|
|
countOnly := false // default: show results
|
|
showLineNumbers := true // default: show chunk indices
|
|
|
|
maxResults := 50
|
|
if mr, ok := args["max_results"].(float64); ok {
|
|
maxResults = int(mr)
|
|
if maxResults < 1 {
|
|
maxResults = 1
|
|
} else if maxResults > 200 {
|
|
maxResults = 200
|
|
}
|
|
}
|
|
|
|
// Parse knowledge_base_ids filter
|
|
var kbIDs []string
|
|
if kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}); ok {
|
|
for _, id := range kbIDsRaw {
|
|
if idStr, ok := id.(string); ok && idStr != "" {
|
|
kbIDs = append(kbIDs, idStr)
|
|
}
|
|
}
|
|
}
|
|
if len(kbIDs) == 0 {
|
|
kbIDs = t.knowledgeBaseIDs
|
|
}
|
|
|
|
// Parse knowledge_ids filter
|
|
var knowledgeIDs []string
|
|
if knowledgeIDsRaw, ok := args["knowledge_ids"].([]interface{}); ok {
|
|
for _, id := range knowledgeIDsRaw {
|
|
if idStr, ok := id.(string); ok && idStr != "" {
|
|
knowledgeIDs = append(knowledgeIDs, idStr)
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.Infof(ctx, "[Tool][GrepChunks] Patterns: %v, MaxResults: %d",
|
|
patterns, maxResults)
|
|
|
|
// Build and execute query
|
|
results, totalCount, err := t.searchChunks(ctx, patterns, kbIDs, knowledgeIDs, maxResults)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "[Tool][GrepChunks] Search failed: %v", err)
|
|
return &types.ToolResult{
|
|
Success: false,
|
|
Error: fmt.Sprintf("Search failed: %v", err),
|
|
}, err
|
|
}
|
|
|
|
logger.Infof(ctx, "[Tool][GrepChunks] Found %d matching chunks", len(results))
|
|
|
|
// Format output
|
|
output := t.formatOutput(ctx, results, totalCount, patterns, contextLines, countOnly, showLineNumbers, kbIDs, knowledgeIDs)
|
|
|
|
return &types.ToolResult{
|
|
Success: true,
|
|
Output: output,
|
|
Data: map[string]interface{}{
|
|
"patterns": patterns,
|
|
"results": results,
|
|
"result_count": len(results),
|
|
"total_matches": totalCount,
|
|
"knowledge_base_ids": kbIDs,
|
|
"knowledge_ids": knowledgeIDs,
|
|
"max_results": maxResults,
|
|
"display_type": "grep_results",
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
// searchChunks performs the database search with pattern matching
|
|
func (t *GrepChunksTool) searchChunks(
|
|
ctx context.Context,
|
|
patterns []string,
|
|
kbIDs []string,
|
|
knowledgeIDs []string,
|
|
maxResults int,
|
|
) ([]map[string]interface{}, int64, error) {
|
|
// Build base query
|
|
query := t.db.WithContext(ctx).Table("chunks").
|
|
Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, chunks.knowledge_base_id, chunks.chunk_type, chunks.created_at, knowledges.title as knowledge_title").
|
|
Joins("LEFT JOIN knowledges ON chunks.knowledge_id = knowledges.id").
|
|
Where("chunks.tenant_id = ?", t.tenantID).
|
|
Where("chunks.is_enabled = ?", true).
|
|
Where("chunks.deleted_at IS NULL").
|
|
Where("knowledges.deleted_at IS NULL")
|
|
|
|
// Apply knowledge base filter
|
|
if len(kbIDs) > 0 {
|
|
query = query.Where("chunks.knowledge_base_id IN ?", kbIDs)
|
|
}
|
|
|
|
// Apply knowledge filter
|
|
if len(knowledgeIDs) > 0 {
|
|
query = query.Where("chunks.knowledge_id IN ?", knowledgeIDs)
|
|
}
|
|
|
|
// Apply pattern matching (case-insensitive fixed string matching, OR logic for multiple patterns)
|
|
if len(patterns) == 1 {
|
|
query = query.Where("chunks.content ILIKE ?", "%"+patterns[0]+"%")
|
|
} else {
|
|
// Multiple patterns: use OR logic
|
|
var conditions []string
|
|
var args []interface{}
|
|
for _, pattern := range patterns {
|
|
conditions = append(conditions, "chunks.content ILIKE ?")
|
|
args = append(args, "%"+pattern+"%")
|
|
}
|
|
query = query.Where("("+strings.Join(conditions, " OR ")+")", args...)
|
|
}
|
|
|
|
// Count total matches first (for count_only mode)
|
|
var totalCount int64
|
|
if err := query.Count(&totalCount).Error; err != nil {
|
|
logger.Warnf(ctx, "[Tool][GrepChunks] Failed to count matches: %v", err)
|
|
}
|
|
|
|
// Fetch results
|
|
var results []map[string]interface{}
|
|
if err := query.Limit(maxResults).Order("chunks.created_at DESC").Find(&results).Error; err != nil {
|
|
logger.Errorf(ctx, "[Tool][GrepChunks] Failed to fetch results: %v", err)
|
|
return nil, 0, err
|
|
}
|
|
|
|
return results, totalCount, nil
|
|
}
|
|
|
|
// formatOutput formats the search results for display (grep-style output)
|
|
func (t *GrepChunksTool) formatOutput(
|
|
ctx context.Context,
|
|
results []map[string]interface{},
|
|
totalCount int64,
|
|
patterns []string,
|
|
contextLines int,
|
|
countOnly bool,
|
|
showLineNumbers bool,
|
|
kbIDs []string,
|
|
knowledgeIDs []string,
|
|
) string {
|
|
var output strings.Builder
|
|
|
|
// If count_only mode, just return the count
|
|
if countOnly {
|
|
output.WriteString(fmt.Sprintf("%d\n", totalCount))
|
|
return output.String()
|
|
}
|
|
|
|
// Show search info
|
|
if len(patterns) == 1 {
|
|
output.WriteString(fmt.Sprintf("Pattern: '%s' (case-insensitive)\n", patterns[0]))
|
|
} else {
|
|
output.WriteString(fmt.Sprintf("Patterns (%d): %v (case-insensitive, OR logic)\n", len(patterns), patterns))
|
|
}
|
|
output.WriteString(fmt.Sprintf("Matches: %d chunk(s)\n\n", len(results)))
|
|
|
|
if len(results) == 0 {
|
|
output.WriteString("No matches found.\n")
|
|
output.WriteString("\n=== ⚠️ CRITICAL - Next Steps ===\n")
|
|
output.WriteString("- ❌ DO NOT use training data or general knowledge to answer\n")
|
|
output.WriteString("- ✅ Try knowledge_search for semantic search\n")
|
|
output.WriteString("- ✅ If KB search fails and web_search is enabled: You MUST use web_search\n")
|
|
output.WriteString("- NEVER fabricate or infer answers - ONLY use retrieved content\n")
|
|
return output.String()
|
|
}
|
|
|
|
// Group by document (like grep showing filename)
|
|
docGroups := make(map[string]struct {
|
|
title string
|
|
chunks []map[string]interface{}
|
|
})
|
|
|
|
for _, result := range results {
|
|
knowledgeID := fmt.Sprintf("%v", result["knowledge_id"])
|
|
title := "Untitled"
|
|
if t := result["knowledge_title"]; t != nil {
|
|
title = fmt.Sprintf("%v", t)
|
|
}
|
|
|
|
group := docGroups[knowledgeID]
|
|
group.title = title
|
|
group.chunks = append(group.chunks, result)
|
|
docGroups[knowledgeID] = group
|
|
}
|
|
|
|
// Display results in grep-style format
|
|
for docID, group := range docGroups {
|
|
// Show document header (like grep showing filename)
|
|
if len(docGroups) > 1 {
|
|
output.WriteString(fmt.Sprintf("\n--- %s (knowledge_id: %s) ---\n", group.title, docID))
|
|
}
|
|
|
|
// Show each matching chunk
|
|
for _, chunk := range group.chunks {
|
|
chunkIndex := 0
|
|
if idx, ok := chunk["chunk_index"]; ok {
|
|
switch v := idx.(type) {
|
|
case int:
|
|
chunkIndex = v
|
|
case int64:
|
|
chunkIndex = int(v)
|
|
case float64:
|
|
chunkIndex = int(v)
|
|
}
|
|
}
|
|
|
|
content := ""
|
|
if c, ok := chunk["content"]; ok && c != nil {
|
|
content = fmt.Sprintf("%v", c)
|
|
}
|
|
|
|
// Extract preview with context around match (case-insensitive)
|
|
// Try each pattern and use the first match
|
|
preview := extractPreviewMultiPattern(content, patterns, false, contextLines)
|
|
|
|
// Format like grep: [filename:]line:content
|
|
if showLineNumbers {
|
|
if len(docGroups) > 1 {
|
|
output.WriteString(fmt.Sprintf("%s:chunk[%d]:%s\n", group.title, chunkIndex, preview))
|
|
} else {
|
|
output.WriteString(fmt.Sprintf("chunk[%d]:%s\n", chunkIndex, preview))
|
|
}
|
|
} else {
|
|
if len(docGroups) > 1 {
|
|
output.WriteString(fmt.Sprintf("%s:%s\n", group.title, preview))
|
|
} else {
|
|
output.WriteString(fmt.Sprintf("%s\n", preview))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add guidance for next steps
|
|
output.WriteString("\n=== Next Steps ===\n")
|
|
if len(results) > 0 {
|
|
output.WriteString("- Found matching documents. Use knowledge_search with semantic queries to understand the context.\n")
|
|
output.WriteString("- Filter knowledge_search by knowledge_ids from above results for better relevance.\n")
|
|
} else {
|
|
output.WriteString("- No matches found. Try different keywords or use knowledge_search for semantic search.\n")
|
|
output.WriteString("- Consider using synonyms or related terms in your search patterns.\n")
|
|
}
|
|
|
|
return output.String()
|
|
}
|
|
|
|
// extractPreview extracts a preview of content around the matched pattern
|
|
func extractPreview(content, pattern string, caseSensitive bool, contextLen int) string {
|
|
if content == "" {
|
|
return ""
|
|
}
|
|
|
|
// Find the pattern in content
|
|
searchContent := content
|
|
searchPattern := pattern
|
|
if !caseSensitive {
|
|
searchContent = strings.ToLower(content)
|
|
searchPattern = strings.ToLower(pattern)
|
|
}
|
|
|
|
pos := strings.Index(searchContent, searchPattern)
|
|
if pos == -1 {
|
|
// If pattern not found (might be regex), just return first 60 chars
|
|
runes := []rune(content)
|
|
if len(runes) <= contextLen*2 {
|
|
return string(runes)
|
|
}
|
|
return string(runes[:contextLen*2])
|
|
}
|
|
|
|
// Extract context around the match
|
|
runes := []rune(content)
|
|
start := pos
|
|
|
|
// Convert to rune positions for proper unicode handling
|
|
runePos := 0
|
|
for i, r := range runes {
|
|
if runePos >= start {
|
|
start = i
|
|
break
|
|
}
|
|
runePos += len(string(r))
|
|
}
|
|
|
|
// Calculate preview bounds
|
|
previewStart := start - contextLen
|
|
if previewStart < 0 {
|
|
previewStart = 0
|
|
}
|
|
|
|
previewEnd := start + len([]rune(pattern)) + contextLen
|
|
if previewEnd > len(runes) {
|
|
previewEnd = len(runes)
|
|
}
|
|
|
|
preview := string(runes[previewStart:previewEnd])
|
|
|
|
// Clean up whitespace
|
|
preview = strings.ReplaceAll(preview, "\n", " ")
|
|
preview = strings.ReplaceAll(preview, "\t", " ")
|
|
|
|
// Collapse multiple spaces
|
|
for strings.Contains(preview, " ") {
|
|
preview = strings.ReplaceAll(preview, " ", " ")
|
|
}
|
|
|
|
return strings.TrimSpace(preview)
|
|
}
|
|
|
|
// extractPreviewMultiPattern extracts a preview around the first matched pattern
|
|
func extractPreviewMultiPattern(content string, patterns []string, caseSensitive bool, contextLen int) string {
|
|
if content == "" || len(patterns) == 0 {
|
|
return ""
|
|
}
|
|
|
|
// Try each pattern and find the first match
|
|
for _, pattern := range patterns {
|
|
preview := extractPreview(content, pattern, caseSensitive, contextLen)
|
|
// Check if this pattern found a match (not just truncated content)
|
|
searchContent := content
|
|
searchPattern := pattern
|
|
if !caseSensitive {
|
|
searchContent = strings.ToLower(content)
|
|
searchPattern = strings.ToLower(pattern)
|
|
}
|
|
if strings.Contains(searchContent, searchPattern) {
|
|
return preview
|
|
}
|
|
}
|
|
|
|
// If no pattern matched, return preview from first pattern
|
|
if len(patterns) > 0 {
|
|
return extractPreview(content, patterns[0], caseSensitive, contextLen)
|
|
}
|
|
|
|
// Fallback: return first N chars
|
|
runes := []rune(content)
|
|
if len(runes) <= contextLen*2 {
|
|
return string(runes)
|
|
}
|
|
return string(runes[:contextLen*2])
|
|
}
|