diff --git a/client/cmd/agent_test/Makefile b/client/cmd/agent_test/Makefile deleted file mode 100644 index dd55bab8..00000000 --- a/client/cmd/agent_test/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -.PHONY: build run clean help - -# Binary name -BINARY=agent_test - -# Build parameters -GOOS?=$(shell go env GOOS) -GOARCH?=$(shell go env GOARCH) - -# Default target -all: build - -# Build the binary -build: - @echo "Building $(BINARY)..." - @go build -o $(BINARY) . - @echo "Build complete: ./$(BINARY)" - -# Build and run with default settings -run: build - @./$(BINARY) - -# Run with custom parameters -run-local: build - @./$(BINARY) -url http://localhost:8080 -kb $(KB_ID) - -# Run with existing session -run-session: build - @./$(BINARY) -url http://localhost:8080 -session $(SESSION_ID) - -# Clean build artifacts -clean: - @echo "Cleaning..." - @rm -f $(BINARY) - @echo "Clean complete" - -# Install dependencies -deps: - @echo "Installing dependencies..." - @go mod download - @go mod tidy - @echo "Dependencies installed" - -# Run tests -test: - @go test -v ./... - -# Build for multiple platforms -build-all: - @echo "Building for multiple platforms..." - @GOOS=linux GOARCH=amd64 go build -o $(BINARY)-linux-amd64 . - @GOOS=darwin GOARCH=amd64 go build -o $(BINARY)-darwin-amd64 . - @GOOS=darwin GOARCH=arm64 go build -o $(BINARY)-darwin-arm64 . - @GOOS=windows GOARCH=amd64 go build -o $(BINARY)-windows-amd64.exe . - @echo "Build complete for all platforms" - -# Help -help: - @echo "Available targets:" - @echo " make build - Build the binary" - @echo " make run - Build and run with no parameters" - @echo " make run-local - Build and run with local server (requires KB_ID env var)" - @echo " make run-session - Build and run with existing session (requires SESSION_ID env var)" - @echo " make clean - Remove build artifacts" - @echo " make deps - Install dependencies" - @echo " make test - Run tests" - @echo " make build-all - Build for multiple platforms" - @echo " make help - Show this help message" - @echo "" - @echo "Examples:" - @echo " make build" - @echo " KB_ID=kb_123 make run-local" - @echo " SESSION_ID=sess_456 make run-session" - @echo " ./$(BINARY) -url http://localhost:8080 -kb kb_123 -token mytoken" - diff --git a/client/cmd/agent_test/QUICKSTART.md b/client/cmd/agent_test/QUICKSTART.md deleted file mode 100644 index cb6eb01f..00000000 --- a/client/cmd/agent_test/QUICKSTART.md +++ /dev/null @@ -1,293 +0,0 @@ -# Quick Start Guide - -Get started with the WeKnora Agent Test Tool in 5 minutes! - -## Prerequisites - -- Go 1.21 or higher -- Access to a running WeKnora instance -- A knowledge base ID (or create one through the tool) - -## Quick Start - -### Option 1: Using the Build Script (Recommended) - -```bash -# 1. Navigate to the tool directory -cd client/cmd/agent_test - -# 2. Build and run with your knowledge base -./run.sh -k kb_your_knowledge_base_id - -# Or with a remote server -./run.sh -u https://api.weknora.com -k kb_12345 -t your_token -``` - -### Option 2: Using Make - -```bash -# Build the binary -make build - -# Run with environment variables -KB_ID=kb_12345 make run-local - -# Or run the binary directly -./agent_test -url http://localhost:8080 -kb kb_12345 -``` - -### Option 3: Direct Go Build - -```bash -# Build -go build -o agent_test . - -# Run -./agent_test -url http://localhost:8080 -kb kb_your_knowledge_base_id -``` - -## First Steps - -Once the tool is running, you'll see an interactive prompt: - -``` -╔════════════════════════════════════════════════════════════╗ -║ WeKnora Agent QA Testing Tool ║ -╚════════════════════════════════════════════════════════════╝ - -[No Session] > -``` - -### 1. Create a New Session - -``` -> new -``` - -The tool will create a new session and show you the session ID. - -### 2. Ask Your First Question - -``` -[Session: abc12345] > ask What is machine learning? -``` - -You'll see real-time output showing: -- 💭 Agent thinking process -- 🔧 Tool calls (if any) -- 📚 Knowledge references -- 📝 Final answer -- 🤔 Reflections (if any) - -### 3. Continue the Conversation - -``` -[Session: abc12345] > ask Can you give me examples? -[Session: abc12345] > ask How does it work? -``` - -### 4. View Session Info - -``` -[Session: abc12345] > info -``` - -### 5. List All Sessions - -``` -[Session: abc12345] > sessions -``` - -### 6. Exit - -``` -[Session: abc12345] > exit -``` - -## Command Reference - -| Command | Description | Example | -|---------|-------------|---------| -| `new` | Create a new session | `> new` | -| `ask ` | Ask the agent a question | `> ask What is AI?` | -| `info` | Show current session details | `> info` | -| `sessions` | List all your sessions | `> sessions` | -| `switch ` | Switch to another session | `> switch sess_123` | -| `help` | Show help message | `> help` | -| `exit` or `quit` | Exit the program | `> exit` | - -## Example Session - -```bash -$ ./run.sh -k kb_ml_docs - -[No Session] > new -✓ Session created successfully! - Session ID: sess_a1b2c3d4 - Knowledge Base: kb_ml_docs - Created At: 2025-11-03T10:00:00Z - -[Session: sess_a1b] > ask What is a neural network? - -╔═══ Agent Processing ═══╗ -Query: What is a neural network? - -💭 Thinking: I need to search the knowledge base for information about neural networks... - -🔧 Tool Call: search_knowledge - Arguments: map[query:neural network definition] - -✓ Tool Result: - Found 3 relevant knowledge chunks - -📚 Knowledge References: Found 3 reference(s) - 1. [Score: 0.892] A neural network is a series of algorithms... - Knowledge: Neural Networks 101 (Chunk: 2) - 2. [Score: 0.856] Neural networks mimic biological neurons... - Knowledge: Deep Learning Basics (Chunk: 5) - 3. [Score: 0.834] The structure consists of layers... - Knowledge: ML Architectures (Chunk: 8) - -📝 Final Answer: -A neural network is a computing system inspired by biological neural networks -in animal brains. It consists of interconnected nodes (artificial neurons) -organized in layers that process information using a connectionist approach -to computation. Neural networks learn to perform tasks by considering examples, -generally without being programmed with task-specific rules. - -🤔 Reflection: I successfully found relevant information in the knowledge base -and provided a comprehensive answer covering the definition, structure, and -learning mechanism of neural networks. - -╔═══ Summary ═══╗ -Duration: 2.34s -Tool Calls: 1 (search_knowledge) -References: 3 -Reflections: 1 -╚═══════════════╝ - -[Session: sess_a1b] > ask What are the main types? - -╔═══ Agent Processing ═══╗ -Query: What are the main types? - -💭 Thinking: The user is asking about types of neural networks based on the -previous context... - -[... continuing the conversation ...] -``` - -## Testing Scenarios - -We provide a test scenarios script for common testing patterns: - -```bash -# Run interactive scenario menu -./test_scenarios.sh - -# Run a specific scenario -./test_scenarios.sh 1 # Basic Q&A -./test_scenarios.sh 2 # Complex queries -./test_scenarios.sh 3 # Multi-turn conversation -``` - -## Configuration with Environment Variables - -Create a configuration file for repeated use: - -```bash -# Create config file -cat > config.env << EOF -export WEKNORA_URL=http://localhost:8080 -export WEKNORA_KB_ID=kb_your_knowledge_base_id -export WEKNORA_TOKEN=your_token_if_needed -EOF - -# Load and run -source config.env -./run.sh -``` - -## Troubleshooting - -### "Connection refused" -- Make sure WeKnora server is running -- Check the URL is correct -- Verify network connectivity - -### "Knowledge base not found" -- Verify your knowledge base ID is correct -- Check you have access to the knowledge base -- Ensure the knowledge base has been initialized - -### "Session not found" -- The session may have expired or been deleted -- Create a new session with `new` command - -### No tool calls happening -- Check your knowledge base has content -- Verify the agent configuration includes tool definitions -- Look for error messages in the output - -## Advanced Usage - -### Using with Docker - -If WeKnora is running in Docker: - -```bash -# Get the container's IP or use host networking -./agent_test -url http://localhost:8080 -kb kb_123 -``` - -### Testing with Production - -```bash -# Use production URL with authentication -./agent_test \ - -url https://api.weknora.com \ - -token $PROD_TOKEN \ - -kb kb_prod_123 -``` - -### Continuing an Existing Session - -```bash -# Skip session creation and continue existing one -./agent_test -url http://localhost:8080 -session sess_existing_id -``` - -### Batch Testing - -```bash -# Pipe commands for automated testing -echo -e "new\nask What is AI?\ninfo\nexit" | ./agent_test -url http://localhost:8080 -kb kb_123 -``` - -## Next Steps - -- Read the full [README.md](README.md) for detailed documentation -- Check [agent_example.go](../../agent_example.go) for code examples -- Explore the [client package documentation](../../README.md) -- Review agent events and response types in the main README - -## Getting Help - -If you encounter issues: - -1. Check the [README.md](README.md) for detailed documentation -2. Review the error messages - they usually indicate the problem -3. Verify your configuration (URL, KB ID, token) -4. Check WeKnora server logs for backend issues -5. Use `info` command to check session state - -## Tips - -- **Start Simple**: Begin with basic questions to verify connectivity -- **Use Info**: The `info` command shows valuable session configuration -- **Watch Events**: Pay attention to all event types to understand agent behavior -- **Check References**: Knowledge references show if retrieval is working correctly -- **Monitor Performance**: The summary shows timing and tool usage statistics - -Happy testing! 🚀 - diff --git a/client/cmd/agent_test/README.md b/client/cmd/agent_test/README.md deleted file mode 100644 index 9c004b3c..00000000 --- a/client/cmd/agent_test/README.md +++ /dev/null @@ -1,307 +0,0 @@ -# WeKnora Agent QA Testing Tool - -An interactive command-line tool for testing and debugging WeKnora's Agent QA functionality. - -## Features - -- 🎯 Interactive terminal interface with colored output -- 📊 Real-time streaming of agent events -- 🔧 Display tool calls and their results -- 💭 Show agent thinking process -- 📚 Display knowledge references with scores -- 🤔 Capture agent reflections -- 📝 Track complete conversation flow -- ⏱️ Performance metrics and timing -- 🔄 Session management (create, switch, list) - -## Installation - -```bash -cd client/cmd/agent_test -go build -o agent_test -``` - -## Usage - -### Basic Usage - -```bash -# Start with a new session -./agent_test -url http://localhost:8080 -kb - -# With authentication token -./agent_test -url http://localhost:8080 -token -kb - -# Use existing session -./agent_test -url http://localhost:8080 -session -``` - -### Command Line Flags - -- `-url` : WeKnora API base URL (default: http://localhost:8080) -- `-token` : API authentication token (optional) -- `-kb` : Knowledge base ID (required for creating new sessions) -- `-session` : Existing session ID to continue conversation (optional) - -### Interactive Commands - -Once the tool is running, you can use these commands: - -#### Session Management - -- `new` - Create a new session - ``` - > new - ``` - -- `sessions` - List all sessions - ``` - > sessions - ``` - -- `switch ` - Switch to another session - ``` - > switch abc123... - ``` - -- `info` - Show current session information - ``` - > info - ``` - -#### Agent Interaction - -- `ask ` - Ask the agent a question - ``` - > ask What is machine learning? - > ask How can I improve model accuracy? - > ask Explain the difference between supervised and unsupervised learning - ``` - -#### General - -- `help` - Show help message -- `exit` or `quit` - Exit the program - -## Agent Events - -The tool displays various agent events in real-time: - -### 💭 Thinking Events -Shows the agent's internal reasoning process -``` -💭 Thinking: Let me search for information about machine learning... -``` - -### 🔧 Tool Call Events -Displays when the agent calls a tool -``` -🔧 Tool Call: search_knowledge - Arguments: {"query": "machine learning definition"} -``` - -### ✓ Tool Result Events -Shows the results from tool execution -``` -✓ Tool Result: - Found 5 relevant knowledge chunks -``` - -### 📚 Knowledge References -Lists retrieved knowledge with relevance scores -``` -📚 Knowledge References: Found 3 reference(s) - 1. [Score: 0.875] Machine learning is a subset of artificial intelligence... - Knowledge: ML Basics (Chunk: 2) - 2. [Score: 0.823] There are three main types of machine learning... - Knowledge: ML Types (Chunk: 5) -``` - -### 📝 Final Answer -The agent's complete response -``` -📝 Final Answer: -Machine learning is an application of artificial intelligence that provides -systems the ability to automatically learn and improve from experience... -``` - -### 🤔 Reflection Events -Agent's self-reflection on its responses -``` -🤔 Reflection: I provided a comprehensive answer covering the main aspects -of machine learning with relevant examples. -``` - -### Summary -At the end of each query, a summary is displayed: -``` -╔═══ Summary ═══╗ -Duration: 3.45s -Tool Calls: 2 (search_knowledge, format_response) -References: 3 -Reflections: 1 -╚═══════════════╝ -``` - -## Example Session - -```bash -$ ./agent_test -url http://localhost:8080 -kb kb_12345 - -╔════════════════════════════════════════════════════════════╗ -║ WeKnora Agent QA Testing Tool ║ -╚════════════════════════════════════════════════════════════╝ - -[No Session] > new -✓ Session created successfully! - Session ID: sess_abc123xyz - Knowledge Base: kb_12345 - Created At: 2025-11-03T10:30:00Z - -[Session: sess_abc] > ask What is neural networks? - -╔═══ Agent Processing ═══╗ -Query: What is neural networks? - -💭 Thinking: I need to search for information about neural networks... - -🔧 Tool Call: search_knowledge - Arguments: map[query:neural networks] - -✓ Tool Result: - Successfully retrieved 3 relevant chunks - -📚 Knowledge References: Found 3 reference(s) - 1. [Score: 0.892] A neural network is a series of algorithms... - Knowledge: Deep Learning Fundamentals (Chunk: 1) - 2. [Score: 0.856] Neural networks consist of layers of neurons... - Knowledge: Neural Network Architecture (Chunk: 3) - -📝 Final Answer: -Neural networks are computing systems inspired by biological neural networks -that constitute animal brains. They consist of interconnected nodes (neurons) -organized in layers... - -🤔 Reflection: The answer provides a clear definition with supporting -references from the knowledge base. - -╔═══ Summary ═══╗ -Duration: 2.87s -Tool Calls: 1 (search_knowledge) -References: 3 -Reflections: 1 -╚═══════════════╝ - -[Session: sess_abc] > info - -╔═══ Session Information ═══╗ -Session ID: sess_abc123xyz -Knowledge Base ID: kb_12345 -Max Rounds: 10 -Created At: 2025-11-03T10:30:00Z -╚═══════════════════════════╝ - -[Session: sess_abc] > exit - -Goodbye! -``` - -## Testing Scenarios - -### 1. Basic Q&A -Test simple question-answering with knowledge retrieval: -``` -> ask What is machine learning? -``` - -### 2. Multi-Turn Conversation -Test context awareness: -``` -> ask What is supervised learning? -> ask Can you give me some examples? -> ask How does it differ from unsupervised learning? -``` - -### 3. Complex Queries -Test agent's reasoning capabilities: -``` -> ask Compare and contrast supervised learning and reinforcement learning, including their use cases -``` - -### 4. Tool Usage -Test tool calling and reflection: -``` -> ask Search for information about deep learning and summarize the key concepts -``` - -### 5. Edge Cases -Test error handling: -``` -> ask [empty query] -> ask [very long query with 1000+ characters] -``` - -## Debugging - -The tool displays detailed information for debugging: - -1. **Event Timing**: Track how long each phase takes -2. **Tool Arguments**: See exactly what parameters are passed to tools -3. **Knowledge Scores**: Evaluate retrieval quality -4. **Error Messages**: Clear error reporting for failed operations -5. **Session State**: Monitor session configuration and status - -## Troubleshooting - -### Connection Issues -``` -✗ Request failed: connection refused -``` -- Check if WeKnora server is running -- Verify the URL is correct -- Check network connectivity - -### Authentication Errors -``` -✗ HTTP error 401: Unauthorized -``` -- Verify your API token is valid -- Check token hasn't expired - -### Session Not Found -``` -✗ Failed to get session info: session not found -``` -- Session may have been deleted -- Create a new session with `new` command - -## Advanced Usage - -### Using with Different Environments - -Development: -```bash -./agent_test -url http://localhost:8080 -kb kb_dev -``` - -Staging: -```bash -./agent_test -url https://staging.weknora.com -token $STAGING_TOKEN -kb kb_staging -``` - -Production: -```bash -./agent_test -url https://api.weknora.com -token $PROD_TOKEN -kb kb_prod -``` - -### Scripting - -You can pipe commands for automated testing: -```bash -echo -e "new\nask What is AI?\nexit" | ./agent_test -url http://localhost:8080 -kb kb_test -``` - -## License - -Copyright (c) 2025 Tencent. All rights reserved. - diff --git a/client/cmd/agent_test/go.mod b/client/cmd/agent_test/go.mod deleted file mode 100644 index 6bbbe051..00000000 --- a/client/cmd/agent_test/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module github.com/Tencent/WeKnora/client/cmd/agent_test - -go 1.24.2 - -require github.com/Tencent/WeKnora/client v0.0.0 - -replace github.com/Tencent/WeKnora/client => ../.. diff --git a/client/cmd/agent_test/main.go b/client/cmd/agent_test/main.go deleted file mode 100644 index ddf0d72d..00000000 --- a/client/cmd/agent_test/main.go +++ /dev/null @@ -1,373 +0,0 @@ -package main - -import ( - "bufio" - "context" - "flag" - "fmt" - "os" - "strings" - "time" - - "github.com/Tencent/WeKnora/client" -) - -// Colors for terminal output -const ( - ColorReset = "\033[0m" - ColorRed = "\033[31m" - ColorGreen = "\033[32m" - ColorYellow = "\033[33m" - ColorBlue = "\033[34m" - ColorMagenta = "\033[35m" - ColorCyan = "\033[36m" - ColorWhite = "\033[37m" - ColorBold = "\033[1m" -) - -// Config holds the configuration -type Config struct { - BaseURL string - Token string - KnowledgeBaseID string -} - -// CLI represents the command line interface -type CLI struct { - client *client.Client - config *Config - currentSessionID string - agentSession *client.AgentSession -} - -func main() { - baseURL := flag.String("url", "http://localhost:8080", "WeKnora API base URL") - token := flag.String("token", "", "API authentication token") - kbID := flag.String("kb", "", "Knowledge base ID") - sessionID := flag.String("session", "", "Existing session ID (optional)") - - flag.Parse() - - config := &Config{ - BaseURL: *baseURL, - Token: *token, - KnowledgeBaseID: *kbID, - } - - cli := NewCLI(config) - - // Use existing session if provided - if *sessionID != "" { - cli.currentSessionID = *sessionID - cli.agentSession = cli.client.NewAgentSession(*sessionID) - fmt.Printf("%s✓ Using existing session: %s%s\n\n", ColorGreen, *sessionID, ColorReset) - } - - cli.Run() -} - -// NewCLI creates a new CLI instance -func NewCLI(config *Config) *CLI { - clientOptions := []client.ClientOption{ - client.WithTimeout(120 * time.Second), - } - if config.Token != "" { - clientOptions = append(clientOptions, client.WithToken(config.Token)) - } - - return &CLI{ - client: client.NewClient(config.BaseURL, clientOptions...), - config: config, - } -} - -// Run starts the interactive CLI -func (cli *CLI) Run() { - cli.printWelcome() - - reader := bufio.NewReader(os.Stdin) - - for { - fmt.Print(cli.getPrompt()) - - input, err := reader.ReadString('\n') - if err != nil { - fmt.Printf("%sError reading input: %v%s\n", ColorRed, err, ColorReset) - continue - } - - input = strings.TrimSpace(input) - if input == "" { - continue - } - - if !cli.handleCommand(input) { - break - } - } - - fmt.Printf("\n%sGoodbye!%s\n", ColorCyan, ColorReset) -} - -func (cli *CLI) getPrompt() string { - if cli.currentSessionID != "" { - return fmt.Sprintf("%s[Session: %s]%s > ", ColorBlue, cli.currentSessionID[:8], ColorReset) - } - return fmt.Sprintf("%s[No Session]%s > ", ColorYellow, ColorReset) -} - -func (cli *CLI) printWelcome() { - fmt.Printf("\n%s╔════════════════════════════════════════════════════════════╗%s\n", ColorCyan, ColorReset) - fmt.Printf("%s║ WeKnora Agent QA Testing Tool ║%s\n", ColorCyan, ColorReset) - fmt.Printf("%s╚════════════════════════════════════════════════════════════╝%s\n\n", ColorCyan, ColorReset) - fmt.Printf("%sAvailable commands:%s\n", ColorBold, ColorReset) - fmt.Printf(" %snew%s - Create a new session\n", ColorGreen, ColorReset) - fmt.Printf(" %sask %s - Ask agent a question\n", ColorGreen, ColorReset) - fmt.Printf(" %sinfo%s - Show current session info\n", ColorGreen, ColorReset) - fmt.Printf(" %ssessions%s - List all sessions\n", ColorGreen, ColorReset) - fmt.Printf(" %sswitch %s - Switch to another session\n", ColorGreen, ColorReset) - fmt.Printf(" %shelp%s - Show this help message\n", ColorGreen, ColorReset) - fmt.Printf(" %sexit/quit%s - Exit the program\n\n", ColorGreen, ColorReset) -} - -func (cli *CLI) handleCommand(input string) bool { - parts := strings.SplitN(input, " ", 2) - command := strings.ToLower(parts[0]) - - switch command { - case "help": - cli.printWelcome() - case "new": - cli.createNewSession() - case "ask": - if len(parts) < 2 { - fmt.Printf("%sUsage: ask %s\n", ColorRed, ColorReset) - return true - } - cli.askQuestion(parts[1]) - case "info": - cli.showSessionInfo() - case "sessions": - cli.listSessions() - case "switch": - if len(parts) < 2 { - fmt.Printf("%sUsage: switch %s\n", ColorRed, ColorReset) - return true - } - cli.switchSession(parts[1]) - case "exit", "quit": - return false - default: - fmt.Printf("%sUnknown command: %s. Type 'help' for available commands.%s\n", ColorRed, command, ColorReset) - } - - return true -} - -func (cli *CLI) createNewSession() { - if cli.config.KnowledgeBaseID == "" { - fmt.Printf("%sError: Knowledge base ID is required. Use -kb flag.%s\n", ColorRed, ColorReset) - return - } - - fmt.Printf("%sCreating new session...%s\n", ColorYellow, ColorReset) - - ctx := context.Background() - request := &client.CreateSessionRequest{ - KnowledgeBaseID: cli.config.KnowledgeBaseID, - SessionStrategy: &client.SessionStrategy{ - MaxRounds: 10, - EnableRewrite: false, - FallbackStrategy: "default", - EmbeddingTopK: 10, - }, - } - - session, err := cli.client.CreateSession(ctx, request) - if err != nil { - fmt.Printf("%s✗ Failed to create session: %v%s\n", ColorRed, err, ColorReset) - return - } - - cli.currentSessionID = session.ID - cli.agentSession = cli.client.NewAgentSession(session.ID) - - fmt.Printf("%s✓ Session created successfully!%s\n", ColorGreen, ColorReset) - fmt.Printf(" Session ID: %s%s%s\n", ColorCyan, session.ID, ColorReset) - fmt.Printf(" Knowledge Base: %s%s%s\n", ColorCyan, session.KnowledgeBaseID, ColorReset) - fmt.Printf(" Created At: %s%s%s\n\n", ColorCyan, session.CreatedAt, ColorReset) -} - -func (cli *CLI) askQuestion(query string) { - if cli.agentSession == nil { - fmt.Printf("%sError: No active session. Create one with 'new' command.%s\n", ColorRed, ColorReset) - return - } - - fmt.Printf("\n%s╔═══ Agent Processing ═══╗%s\n", ColorCyan, ColorReset) - fmt.Printf("%sQuery: %s%s%s\n\n", ColorBold, ColorWhite, query, ColorReset) - - ctx := context.Background() - - var ( - thinkingContent string - finalAnswer string - references []*client.SearchResult - toolCalls []string - reflections []string - ) - - startTime := time.Now() - - err := cli.agentSession.Ask(ctx, query, func(resp *client.AgentStreamResponse) error { - switch resp.ResponseType { - case client.AgentResponseTypeThinking: - thinkingContent += resp.Content - if resp.Done { - fmt.Printf("%s💭 Thinking:%s %s\n\n", ColorYellow, ColorReset, thinkingContent) - thinkingContent = "" - } - - case client.AgentResponseTypeToolCall: - toolName := "Unknown" - if resp.Data != nil { - if name, ok := resp.Data["tool_name"].(string); ok { - toolName = name - } - } - toolCalls = append(toolCalls, toolName) - fmt.Printf("%s🔧 Tool Call:%s %s\n", ColorMagenta, ColorReset, toolName) - if resp.Data != nil { - if args, ok := resp.Data["arguments"]; ok { - fmt.Printf(" Arguments: %v\n", args) - } - } - - case client.AgentResponseTypeToolResult: - fmt.Printf("%s✓ Tool Result:%s\n", ColorGreen, ColorReset) - fmt.Printf(" %s\n\n", resp.Content) - - case client.AgentResponseTypeReferences: - if resp.KnowledgeReferences != nil { - references = append(references, resp.KnowledgeReferences...) - fmt.Printf( - "%s📚 Knowledge References:%s Found %d reference(s)\n", - ColorCyan, - ColorReset, - len(resp.KnowledgeReferences), - ) - for i, ref := range resp.KnowledgeReferences { - fmt.Printf(" %d. [Score: %.3f] %s\n", i+1, ref.Score, truncateString(ref.Content, 80)) - fmt.Printf(" Knowledge: %s (Chunk: %d)\n", ref.KnowledgeTitle, ref.ChunkIndex) - } - fmt.Println() - } - - case client.AgentResponseTypeAnswer: - finalAnswer += resp.Content - if resp.Done { - fmt.Printf("%s📝 Final Answer:%s\n", ColorGreen, ColorReset) - fmt.Printf("%s\n\n", finalAnswer) - } - - case client.AgentResponseTypeReflection: - if resp.Done && resp.Content != "" { - reflections = append(reflections, resp.Content) - fmt.Printf("%s🤔 Reflection:%s %s\n\n", ColorBlue, ColorReset, resp.Content) - } - - case client.AgentResponseTypeError: - fmt.Printf("%s✗ Error:%s %s\n\n", ColorRed, ColorReset, resp.Content) - } - - return nil - }) - if err != nil { - fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset) - return - } - - duration := time.Since(startTime) - - // Print summary - fmt.Printf("%s╔═══ Summary ═══╗%s\n", ColorCyan, ColorReset) - fmt.Printf("Duration: %s%.2fs%s\n", ColorCyan, duration.Seconds(), ColorReset) - fmt.Printf("Tool Calls: %s%d%s", ColorCyan, len(toolCalls), ColorReset) - if len(toolCalls) > 0 { - fmt.Printf(" (%s)", strings.Join(toolCalls, ", ")) - } - fmt.Println() - fmt.Printf("References: %s%d%s\n", ColorCyan, len(references), ColorReset) - fmt.Printf("Reflections: %s%d%s\n", ColorCyan, len(reflections), ColorReset) - fmt.Printf("%s╚═══════════════╝%s\n\n", ColorCyan, ColorReset) -} - -func (cli *CLI) showSessionInfo() { - if cli.currentSessionID == "" { - fmt.Printf("%sNo active session.%s\n", ColorYellow, ColorReset) - return - } - - ctx := context.Background() - session, err := cli.client.GetSession(ctx, cli.currentSessionID) - if err != nil { - fmt.Printf("%s✗ Failed to get session info: %v%s\n", ColorRed, err, ColorReset) - return - } - - fmt.Printf("\n%s╔═══ Session Information ═══╗%s\n", ColorCyan, ColorReset) - fmt.Printf("Session ID: %s%s%s\n", ColorWhite, session.ID, ColorReset) - fmt.Printf("Title: %s%s%s\n", ColorWhite, session.Title, ColorReset) - fmt.Printf("Knowledge Base ID: %s%s%s\n", ColorWhite, session.KnowledgeBaseID, ColorReset) - fmt.Printf("Max Rounds: %s%d%s\n", ColorWhite, session.MaxRounds, ColorReset) - fmt.Printf("Enable Rewrite: %s%v%s\n", ColorWhite, session.EnableRewrite, ColorReset) - fmt.Printf("Summary Model: %s%s%s\n", ColorWhite, session.SummaryModelID, ColorReset) - fmt.Printf("Created At: %s%s%s\n", ColorWhite, session.CreatedAt, ColorReset) - fmt.Printf("Updated At: %s%s%s\n", ColorWhite, session.UpdatedAt, ColorReset) - fmt.Printf("%s╚═══════════════════════════╝%s\n\n", ColorCyan, ColorReset) -} - -func (cli *CLI) listSessions() { - ctx := context.Background() - sessions, total, err := cli.client.GetSessionsByTenant(ctx, 1, 10) - if err != nil { - fmt.Printf("%s✗ Failed to list sessions: %v%s\n", ColorRed, err, ColorReset) - return - } - - fmt.Printf("\n%s╔═══ Sessions (Total: %d) ═══╗%s\n", ColorCyan, total, ColorReset) - for i, session := range sessions { - marker := " " - if session.ID == cli.currentSessionID { - marker = "►" - } - fmt.Printf("%s%s %d. %s%s\n", ColorGreen, marker, i+1, session.ID, ColorReset) - fmt.Printf(" Title: %s\n", session.Title) - fmt.Printf(" KB: %s\n", session.KnowledgeBaseID) - fmt.Printf(" Created: %s\n", session.CreatedAt) - fmt.Println() - } - fmt.Printf("%s╚════════════════════════════╝%s\n\n", ColorCyan, ColorReset) -} - -func (cli *CLI) switchSession(sessionID string) { - ctx := context.Background() - session, err := cli.client.GetSession(ctx, sessionID) - if err != nil { - fmt.Printf("%s✗ Failed to switch session: %v%s\n", ColorRed, err, ColorReset) - return - } - - cli.currentSessionID = session.ID - cli.agentSession = cli.client.NewAgentSession(session.ID) - - fmt.Printf("%s✓ Switched to session: %s%s\n", ColorGreen, session.ID, ColorReset) -} - -func truncateString(s string, maxLen int) string { - s = strings.ReplaceAll(s, "\n", " ") - if len(s) <= maxLen { - return s - } - return s[:maxLen] + "..." -} diff --git a/client/cmd/agent_test/run.sh b/client/cmd/agent_test/run.sh deleted file mode 100755 index 906ed5af..00000000 --- a/client/cmd/agent_test/run.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/bash - -# WeKnora Agent Test Tool - Quick Start Script -# This script helps you quickly build and run the agent test tool - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Default values -DEFAULT_URL="http://localhost:8080" -BINARY_NAME="agent_test" - -# Function to print colored output -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} - -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -# Function to build the binary -build() { - print_info "Building agent test tool..." - if go build -o "$BINARY_NAME" .; then - print_success "Build successful!" - return 0 - else - print_error "Build failed!" - return 1 - fi -} - -# Function to show usage -usage() { - cat << EOF -WeKnora Agent Test Tool - Quick Start Script - -Usage: - $0 [OPTIONS] - -Options: - -u, --url URL WeKnora API URL (default: $DEFAULT_URL) - -k, --kb KB_ID Knowledge Base ID - -s, --session ID Existing Session ID - -t, --token TOKEN API authentication token - -b, --build-only Only build, don't run - -h, --help Show this help message - -Examples: - # Build and run with a knowledge base - $0 -k kb_123456 - - # Use a different server URL - $0 -u https://api.weknora.com -k kb_123456 -t mytoken - - # Continue an existing session - $0 -s sess_123456 - - # Only build the binary - $0 --build-only - -Environment Variables: - WEKNORA_URL Default API URL - WEKNORA_KB_ID Default Knowledge Base ID - WEKNORA_TOKEN Default API Token - WEKNORA_SESSION_ID Default Session ID - -EOF -} - -# Parse command line arguments -URL="${WEKNORA_URL:-$DEFAULT_URL}" -KB_ID="${WEKNORA_KB_ID:-}" -SESSION_ID="${WEKNORA_SESSION_ID:-}" -TOKEN="${WEKNORA_TOKEN:-}" -BUILD_ONLY=false - -while [[ $# -gt 0 ]]; do - case $1 in - -u|--url) - URL="$2" - shift 2 - ;; - -k|--kb) - KB_ID="$2" - shift 2 - ;; - -s|--session) - SESSION_ID="$2" - shift 2 - ;; - -t|--token) - TOKEN="$2" - shift 2 - ;; - -b|--build-only) - BUILD_ONLY=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - print_error "Unknown option: $1" - usage - exit 1 - ;; - esac -done - -# Build the binary -if ! build; then - exit 1 -fi - -# Exit if build-only mode -if [ "$BUILD_ONLY" = true ]; then - print_info "Build-only mode. Binary created: ./$BINARY_NAME" - exit 0 -fi - -# Validate parameters -if [ -z "$SESSION_ID" ] && [ -z "$KB_ID" ]; then - print_warning "Neither session ID nor knowledge base ID provided." - print_info "You can create a new session after starting the tool." -fi - -# Build command arguments -ARGS="-url $URL" - -if [ -n "$KB_ID" ]; then - ARGS="$ARGS -kb $KB_ID" -fi - -if [ -n "$SESSION_ID" ]; then - ARGS="$ARGS -session $SESSION_ID" -fi - -if [ -n "$TOKEN" ]; then - ARGS="$ARGS -token $TOKEN" -fi - -# Show configuration -echo "" -print_info "Configuration:" -echo " URL: $URL" -[ -n "$KB_ID" ] && echo " Knowledge Base: $KB_ID" -[ -n "$SESSION_ID" ] && echo " Session: $SESSION_ID" -[ -n "$TOKEN" ] && echo " Token: [provided]" -echo "" - -# Run the tool -print_info "Starting agent test tool..." -echo "" -./$BINARY_NAME $ARGS - diff --git a/client/cmd/agent_test/test_scenarios.sh b/client/cmd/agent_test/test_scenarios.sh deleted file mode 100755 index 385b041b..00000000 --- a/client/cmd/agent_test/test_scenarios.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash - -# Test Scenarios for WeKnora Agent QA -# This script contains various test scenarios you can run - -set -e - -# Colors -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' - -print_scenario() { - echo -e "\n${BLUE}═══════════════════════════════════════════════${NC}" - echo -e "${BLUE} $1${NC}" - echo -e "${BLUE}═══════════════════════════════════════════════${NC}\n" -} - -# Check if binary exists -if [ ! -f "./agent_test" ]; then - echo "Building agent_test..." - make build -fi - -# Load environment variables if .env exists -if [ -f ".env" ]; then - export $(cat .env | grep -v '^#' | xargs) -fi - -# Check required variables -if [ -z "$WEKNORA_KB_ID" ]; then - echo "Error: WEKNORA_KB_ID is not set" - echo "Please set it in .env file or export WEKNORA_KB_ID=your_kb_id" - exit 1 -fi - -URL="${WEKNORA_URL:-http://localhost:8080}" -KB_ID="$WEKNORA_KB_ID" -TOKEN="${WEKNORA_TOKEN:-}" - -# Scenario 1: Basic Q&A -scenario_basic() { - print_scenario "Scenario 1: Basic Question & Answer" - echo -e "${YELLOW}This will test basic agent Q&A functionality${NC}\n" - echo "Try asking: What is machine learning?" - echo "" - - if [ -n "$TOKEN" ]; then - ./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN" - else - ./agent_test -url "$URL" -kb "$KB_ID" - fi -} - -# Scenario 2: Complex query with tool usage -scenario_complex() { - print_scenario "Scenario 2: Complex Query with Tool Usage" - echo -e "${YELLOW}This will test agent's ability to use tools${NC}\n" - echo "Try asking: Search for information about neural networks and explain the key concepts" - echo "" - - if [ -n "$TOKEN" ]; then - ./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN" - else - ./agent_test -url "$URL" -kb "$KB_ID" - fi -} - -# Scenario 3: Multi-turn conversation -scenario_multiturn() { - print_scenario "Scenario 3: Multi-turn Conversation" - echo -e "${YELLOW}This will test agent's context awareness${NC}\n" - echo "Try this sequence:" - echo " 1. What is supervised learning?" - echo " 2. Can you give me some examples?" - echo " 3. How does it compare to unsupervised learning?" - echo "" - - if [ -n "$TOKEN" ]; then - ./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN" - else - ./agent_test -url "$URL" -kb "$KB_ID" - fi -} - -# Show menu -show_menu() { - echo -e "\n${GREEN}WeKnora Agent Test Scenarios${NC}\n" - echo "Select a test scenario to run:" - echo " 1) Basic Q&A" - echo " 2) Complex Query with Tool Usage" - echo " 3) Multi-turn Conversation" - echo " 4) Custom (manual testing)" - echo " 5) Exit" - echo "" - read -p "Enter your choice [1-5]: " choice - - case $choice in - 1) scenario_basic ;; - 2) scenario_complex ;; - 3) scenario_multiturn ;; - 4) - print_scenario "Custom Testing" - if [ -n "$TOKEN" ]; then - ./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN" - else - ./agent_test -url "$URL" -kb "$KB_ID" - fi - ;; - 5) - echo "Goodbye!" - exit 0 - ;; - *) - echo "Invalid choice. Please try again." - show_menu - ;; - esac -} - -# Main -if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then - echo "Usage: $0 [scenario_number]" - echo "" - echo "Available scenarios:" - echo " 1 - Basic Q&A" - echo " 2 - Complex Query with Tool Usage" - echo " 3 - Multi-turn Conversation" - echo " 4 - Custom (manual testing)" - echo "" - echo "If no argument is provided, an interactive menu will be shown." - exit 0 -fi - -# If scenario number is provided as argument -if [ -n "$1" ]; then - case $1 in - 1) scenario_basic ;; - 2) scenario_complex ;; - 3) scenario_multiturn ;; - 4) - if [ -n "$TOKEN" ]; then - ./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN" - else - ./agent_test -url "$URL" -kb "$KB_ID" - fi - ;; - *) - echo "Invalid scenario number: $1" - echo "Use --help to see available scenarios" - exit 1 - ;; - esac -else - # Show interactive menu - show_menu -fi - diff --git a/internal/agent/const.go b/internal/agent/const.go index 120b5000..0d0109da 100644 --- a/internal/agent/const.go +++ b/internal/agent/const.go @@ -1,8 +1,12 @@ package agent const ( - DefaultAgentTemperature = 0.7 - DefaultAgentMaxIterations = 20 + // DefaultAgentTemperature is the default temperature for the agent + DefaultAgentTemperature = 0.7 + // DefaultAgentMaxIterations is the default maximum number of iterations for the agent + DefaultAgentMaxIterations = 20 + // DefaultAgentReflectionEnabled is the default whether to enable reflection for the agent DefaultAgentReflectionEnabled = false - DefaultUseCustomSystemPrompt = false + // DefaultUseCustomSystemPrompt is the default whether to use custom system prompt for the agent + DefaultUseCustomSystemPrompt = false ) diff --git a/internal/agent/engine.go b/internal/agent/engine.go index c6480879..e0364ade 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -96,7 +96,7 @@ func (e *AgentEngine) Execute( } // Build system prompt using progressive RAG prompt - systemPrompt := BuildProgressiveRAGSystemPrompt( + systemPrompt := BuildSystemPrompt( e.knowledgeBasesInfo, e.config.WebSearchEnabled, e.systemPromptTemplate, @@ -771,7 +771,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus( }) // Build messages with all context - systemPrompt := BuildProgressiveRAGSystemPrompt( + systemPrompt := BuildSystemPrompt( e.knowledgeBasesInfo, e.config.WebSearchEnabled, e.systemPromptTemplate, diff --git a/internal/agent/prompts.go b/internal/agent/prompts.go index 2f0bb9b2..0ad9d580 100644 --- a/internal/agent/prompts.go +++ b/internal/agent/prompts.go @@ -209,8 +209,8 @@ func renderPromptPlaceholdersWithStatus( return result } -// BuildProgressiveRAGSystemPromptWithWeb builds the progressive RAG system prompt with web search enabled -func BuildProgressiveRAGSystemPromptWithWeb( +// BuildSystemPromptWithWeb builds the progressive RAG system prompt with web search enabled +func BuildSystemPromptWithWeb( knowledgeBases []*KnowledgeBaseInfo, systemPromptTemplate ...string, ) string { @@ -224,8 +224,8 @@ func BuildProgressiveRAGSystemPromptWithWeb( return renderPromptPlaceholdersWithStatus(template, knowledgeBases, true, currentTime) } -// BuildProgressiveRAGSystemPromptWithoutWeb builds the progressive RAG system prompt without web search -func BuildProgressiveRAGSystemPromptWithoutWeb( +// BuildSystemPromptWithoutWeb builds the progressive RAG system prompt without web search +func BuildSystemPromptWithoutWeb( knowledgeBases []*KnowledgeBaseInfo, systemPromptTemplate ...string, ) string { @@ -241,15 +241,15 @@ func BuildProgressiveRAGSystemPromptWithoutWeb( // BuildProgressiveRAGSystemPrompt builds the progressive RAG system prompt based on web search status // This is the main function to use - it automatically selects the appropriate version -func BuildProgressiveRAGSystemPrompt( +func BuildSystemPrompt( knowledgeBases []*KnowledgeBaseInfo, webSearchEnabled bool, systemPromptTemplate ...string, ) string { if webSearchEnabled { - return BuildProgressiveRAGSystemPromptWithWeb(knowledgeBases, systemPromptTemplate...) + return BuildSystemPromptWithWeb(knowledgeBases, systemPromptTemplate...) } - return BuildProgressiveRAGSystemPromptWithoutWeb(knowledgeBases, systemPromptTemplate...) + return BuildSystemPromptWithoutWeb(knowledgeBases, systemPromptTemplate...) } // ProgressiveRAGSystemPromptWithWeb is the progressive RAG system prompt template with web search enabled diff --git a/internal/agent/tools/get_document_info.go b/internal/agent/tools/get_document_info.go index da8b4c94..45579734 100644 --- a/internal/agent/tools/get_document_info.go +++ b/internal/agent/tools/get_document_info.go @@ -98,6 +98,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter } } + // Check if knowledge IDs are valid if len(knowledgeIDs) == 0 { return &types.ToolResult{ Success: false, @@ -116,6 +117,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter var mu sync.Mutex results := make(map[string]*docInfo) + // Concurrently get info for each knowledge ID for _, knowledgeID := range knowledgeIDs { wg.Add(1) go func(id string) { @@ -195,10 +197,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter for i, doc := range successDocs { k := doc.knowledge - // Get file type icon - typeIcon := getTypeIcon(k.Type, k.FileType) - - output += fmt.Sprintf("【文档 #%d】 %s\n", i+1, typeIcon) + output += fmt.Sprintf("【文档 #%d】\n", i+1) output += fmt.Sprintf(" ID: %s\n", k.ID) output += fmt.Sprintf(" 标题: %s\n", k.Title) @@ -240,7 +239,6 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter "parse_status": k.ParseStatus, "chunk_count": doc.chunkCount, "metadata": k.GetMetadata(), - "type_icon": typeIcon, }) } @@ -264,36 +262,6 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter }, nil } -// Helper functions - -func getTypeIcon(knowledgeType, fileType string) string { - switch knowledgeType { - case "file": - switch fileType { - case "pdf", "PDF": - return "📄 PDF" - case "docx", "doc", "DOCX", "DOC": - return "📝 Word" - case "txt", "TXT": - return "📃 Text" - case "md", "MD", "markdown": - return "📋 Markdown" - case "xlsx", "xls", "XLSX", "XLS": - return "📊 Excel" - case "pptx", "ppt", "PPTX", "PPT": - return "📊 PowerPoint" - default: - return "📄 File" - } - case "url": - return "🌐 URL" - case "passage": - return "📝 Passage" - default: - return "📄 Document" - } -} - func formatSource(knowledgeType, source string) string { switch knowledgeType { case "file": diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index 7e852c60..0a546cf9 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -1173,44 +1173,6 @@ func (t *KnowledgeSearchTool) formatOutput( output += fmt.Sprintf(" 已召回: %d 个 (%.1f%%)\n", retrievedCount, percentage) output += fmt.Sprintf(" 未召回: %d 个\n", remaining) - if remaining > 0 { - output += " 建议: 使用 list_knowledge_chunks 工具获取完整内容\n" - - // Find missing chunk ranges (gaps in retrieved chunks) - missingRanges := t.findMissingChunkRanges(retrievedChunks, int(totalChunks)) - - if len(missingRanges) == 0 { - // No gaps found (shouldn't happen if remaining > 0, but handle it) - output += fmt.Sprintf( - " - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", - knowledgeID, - totalChunks, - ) - } else if len(missingRanges) == 1 && missingRanges[0].start == 0 && missingRanges[0].end == int(totalChunks)-1 { - // All chunks are missing (shouldn't happen, but handle it) - output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks) - } else { - // Suggest getting each missing range - for idx, r := range missingRanges { - rangeSize := r.end - r.start + 1 - if rangeSize <= 100 { - // Small range, get all at once - output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个) → list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=%d)\n", - idx+1, r.start, r.end, rangeSize, knowledgeID, r.start, rangeSize) - } else { - // Large range, suggest getting in batches - output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个,建议分批获取):\n", - idx+1, r.start, r.end, rangeSize) - output += fmt.Sprintf(" 首次: list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=100)\n", - knowledgeID, r.start) - if rangeSize > 100 { - output += " 继续: 根据返回结果调整 offset 继续获取剩余内容\n" - } - } - } - } - } - output += "\n" } } diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index c6e8d183..761ea99e 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -168,6 +168,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i }, nil } +// lookupKnowledgeTitle looks up the title of a knowledge document func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, knowledgeID string) string { if t.knowledgeService == nil { return "" @@ -179,6 +180,7 @@ func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, know return strings.TrimSpace(knowledge.Title) } +// buildOutput builds the output for the list knowledge chunks tool func (t *ListKnowledgeChunksTool) buildOutput( knowledgeID string, knowledgeTitle string, @@ -226,6 +228,7 @@ func (t *ListKnowledgeChunksTool) buildOutput( return builder.String() } +// summarizeContent summarizes the content of a chunk func summarizeContent(content string) string { cleaned := strings.TrimSpace(content) if cleaned == "" { diff --git a/internal/agent/tools/query_knowledge_graph.go b/internal/agent/tools/query_knowledge_graph.go index 21013610..2a58a0b9 100644 --- a/internal/agent/tools/query_knowledge_graph.go +++ b/internal/agent/tools/query_knowledge_graph.go @@ -85,7 +85,7 @@ func (t *QueryKnowledgeGraphTool) Parameters() map[string]interface{} { } } -// Execute queries the knowledge graph with concurrent KB processing +// Execute performs the knowledge graph query with concurrent KB processing func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { // Extract knowledge_base_ids array kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}) @@ -394,14 +394,10 @@ func buildGraphVisualizationData( } } - // TODO: Extract actual entities and relations when graph extraction is fully implemented - // For now, create placeholder structure - return map[string]interface{}{ - "nodes": nodes, - "edges": edges, - "total_nodes": len(nodes), - "total_edges": len(edges), - "visualization_note": "完整的图可视化功能开发中,当前显示文档节点", + "nodes": nodes, + "edges": edges, + "total_nodes": len(nodes), + "total_edges": len(edges), } } diff --git a/internal/agent/tools/sequentialthinking.go b/internal/agent/tools/sequentialthinking.go index 5158f845..c0a8241d 100644 --- a/internal/agent/tools/sequentialthinking.go +++ b/internal/agent/tools/sequentialthinking.go @@ -8,7 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// SequentialThinkingTool implements a dynamic and reflective problem-solving tool +// SequentialThinkingTool is a dynamic and reflective problem-solving tool // This tool helps analyze problems through a flexible thinking process that can adapt and evolve type SequentialThinkingTool struct { BaseTool diff --git a/internal/agent/tools/web_fetch.go b/internal/agent/tools/web_fetch.go index 7bdd952c..449645df 100644 --- a/internal/agent/tools/web_fetch.go +++ b/internal/agent/tools/web_fetch.go @@ -19,29 +19,31 @@ import ( ) const ( - webFetchTimeout = 60 * time.Second - webFetchMaxChars = 100000 + webFetchTimeout = 60 * time.Second // timeout for web fetch + webFetchMaxChars = 100000 // maximum number of characters to fetch ) +// webFetchParams is the parameters for the web fetch tool type webFetchParams struct { URL string Prompt string } +// webFetchItemResult is the result for a web fetch item type webFetchItemResult struct { output string data map[string]interface{} err error } -// WebFetchTool 拉取网页内容并根据提示执行总结 +// WebFetchTool fetches web page content and summarizes it using an LLM type WebFetchTool struct { BaseTool client *http.Client chatModel chat.Chat } -// NewWebFetchTool 创建 web_fetch 工具实例 +// NewWebFetchTool creates a new web_fetch tool instance func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool { description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM. @@ -226,6 +228,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) }, nil } +// parseParams parses the parameters for a web fetch item func (t *WebFetchTool) parseParams(item interface{}) webFetchParams { params := webFetchParams{} if m, ok := item.(map[string]interface{}); ok { @@ -239,6 +242,7 @@ func (t *WebFetchTool) parseParams(item interface{}) webFetchParams { return params } +// validateParams validates the parameters for a web fetch item func (t *WebFetchTool) validateParams(p webFetchParams) error { if p.URL == "" { return fmt.Errorf("url is required") @@ -252,6 +256,7 @@ func (t *WebFetchTool) validateParams(p webFetchParams) error { return nil } +// executeFetch executes a web fetch item func (t *WebFetchTool) executeFetch( ctx context.Context, params webFetchParams, @@ -294,6 +299,7 @@ func (t *WebFetchTool) executeFetch( return output, resultData, summaryErr } +// normalizeGitHubURL normalizes a GitHub URL func (t *WebFetchTool) normalizeGitHubURL(source string) string { if strings.Contains(source, "github.com") && strings.Contains(source, "/blob/") { source = strings.Replace(source, "github.com", "raw.githubusercontent.com", 1) @@ -302,6 +308,7 @@ func (t *WebFetchTool) normalizeGitHubURL(source string) string { return source } +// processWithLLM processes the content with an LLM func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams, content string) (string, error) { if t.chatModel == nil { return "", fmt.Errorf("chat model not available for web_fetch") @@ -336,6 +343,7 @@ func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams return strings.TrimSpace(response.Content), nil } +// buildOutputText builds the output text for a web fetch item func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, summary string, summaryErr error) string { var builder strings.Builder builder.WriteString(fmt.Sprintf("URL: %s\n", params.URL)) @@ -354,6 +362,7 @@ func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, su return builder.String() } +// fetchHTMLContent fetches the HTML content for a web fetch item func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) { html, err := t.fetchWithChromedp(ctx, targetURL) if err == nil && strings.TrimSpace(html) != "" { @@ -375,6 +384,7 @@ func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) ( return html, "http", nil } +// fetchWithChromedp fetches the HTML content with Chromedp func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) { logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL) @@ -414,6 +424,7 @@ func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) return html, nil } +// fetchWithHTTP fetches the HTML content with HTTP func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) { resp, err := t.fetchWithTimeout(ctx, targetURL) if err != nil { @@ -434,6 +445,7 @@ func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (str return string(htmlBytes), nil } +// fetchWithTimeout fetches the HTML content with a timeout func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) if err != nil { @@ -451,6 +463,7 @@ func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) ( return t.client.Do(req) } +// convertHTMLToText converts the HTML content to text func (t *WebFetchTool) convertHTMLToText(html string) string { doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) if err != nil { @@ -469,6 +482,7 @@ func (t *WebFetchTool) convertHTMLToText(html string) string { return strings.TrimSpace(result) } +// processNode processes a node in the HTML content func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Builder) { s.Contents().Each(func(i int, node *goquery.Selection) { nodeName := goquery.NodeName(node) @@ -578,6 +592,7 @@ func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Build }) } +// basicTextExtraction extracts the text from the HTML content func (t *WebFetchTool) basicTextExtraction(html string) string { re := regexp.MustCompile(`<[^>]*>`) text := re.ReplaceAllString(html, " ") diff --git a/internal/application/repository/retriever/neo4j/repository.go b/internal/application/repository/retriever/neo4j/repository.go index 20e9f7fb..0c563ff1 100644 --- a/internal/application/repository/retriever/neo4j/repository.go +++ b/internal/application/repository/retriever/neo4j/repository.go @@ -11,19 +11,23 @@ import ( "github.com/neo4j/neo4j-go-driver/v6/neo4j" ) +// Neo4jRepository is a repository for Neo4j type Neo4jRepository struct { driver neo4j.Driver nodePrefix string } +// NewNeo4jRepository creates a new Neo4j repository func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository { return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"} } +// _remove_hyphen removes hyphens from a string func _remove_hyphen(s string) string { return strings.ReplaceAll(s, "-", "_") } +// Labels returns the labels for a namespace func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string { res := make([]string, 0) for _, label := range namespace.Labels() { @@ -32,12 +36,13 @@ func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string { return res } +// Label returns the label for a namespace func (n *Neo4jRepository) Label(namespace types.NameSpace) string { labels := n.Labels(namespace) return strings.Join(labels, ":") } -// AddGraph implements interfaces.RetrieveGraphRepository. +// AddGraph adds a graph to the Neo4j repository func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error { if n.driver == nil { logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH") @@ -51,11 +56,13 @@ func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpac return nil } +// addGraph adds a graph to the Neo4j repository func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error { session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) defer session.Close(ctx) _, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) { + // Node import query node_import_query := ` UNWIND $data AS row CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node @@ -76,6 +83,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac return nil, fmt.Errorf("failed to create nodes: %v", err) } + // Relationship import query rel_import_query := ` UNWIND $data AS row CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source @@ -106,7 +114,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac return nil } -// DelGraph implements interfaces.RetrieveGraphRepository. +// DelGraph deletes a graph from the Neo4j repository func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error { if n.driver == nil { logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH") @@ -152,6 +160,7 @@ func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameS return nil } +// SearchNode searches for nodes in the Neo4j repository func (n *Neo4jRepository) SearchNode( ctx context.Context, namespace types.NameSpace, diff --git a/internal/application/repository/tag.go b/internal/application/repository/tag.go index 18af8524..20c6687f 100644 --- a/internal/application/repository/tag.go +++ b/internal/application/repository/tag.go @@ -8,6 +8,7 @@ import ( "gorm.io/gorm" ) +// knowledgeTagRepository is a repository for knowledge tags type knowledgeTagRepository struct { db *gorm.DB } @@ -17,14 +18,17 @@ func NewKnowledgeTagRepository(db *gorm.DB) interfaces.KnowledgeTagRepository { return &knowledgeTagRepository{db: db} } +// Create creates a new knowledge tag func (r *knowledgeTagRepository) Create(ctx context.Context, tag *types.KnowledgeTag) error { return r.db.WithContext(ctx).Create(tag).Error } +// Update updates a knowledge tag func (r *knowledgeTagRepository) Update(ctx context.Context, tag *types.KnowledgeTag) error { return r.db.WithContext(ctx).Save(tag).Error } +// GetByID gets a knowledge tag by ID func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.KnowledgeTag, error) { var tag types.KnowledgeTag if err := r.db.WithContext(ctx). @@ -35,6 +39,7 @@ func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, i return &tag, nil } +// ListByKB lists knowledge tags by knowledge base ID func (r *knowledgeTagRepository) ListByKB( ctx context.Context, tenantID uint64, @@ -50,13 +55,14 @@ func (r *knowledgeTagRepository) ListByKB( return tags, nil } +// Delete deletes a knowledge tag func (r *knowledgeTagRepository) Delete(ctx context.Context, tenantID uint64, id string) error { return r.db.WithContext(ctx). Where("tenant_id = ? AND id = ?", tenantID, id). Delete(&types.KnowledgeTag{}).Error } -// CountReferences returns how many knowledges and chunks reference this tag. +// CountReferences returns the number of knowledges and chunks that reference this tag func (r *knowledgeTagRepository) CountReferences( ctx context.Context, tenantID uint64, diff --git a/internal/application/service/chat_pipline/common.go b/internal/application/service/chat_pipline/common.go index cfa247f8..cf28ff6a 100644 --- a/internal/application/service/chat_pipline/common.go +++ b/internal/application/service/chat_pipline/common.go @@ -10,31 +10,32 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" ) +// pipelineInfo logs pipeline info level entries. func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) { common.PipelineInfo(ctx, stage, action, fields) } +// pipelineWarn logs pipeline warning level entries. func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) { common.PipelineWarn(ctx, stage, action, fields) } +// pipelineError logs pipeline error level entries. func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) { common.PipelineError(ctx, stage, action, fields) } // prepareChatModel shared logic to prepare chat model and options +// it gets the chat model and sets up the chat options based on the chat manage. func prepareChatModel(ctx context.Context, modelService interfaces.ModelService, chatManage *types.ChatManage, ) (chat.Chat, *chat.ChatOptions, error) { - logger.Infof(ctx, "Getting chat model, model ID: %s", chatManage.ChatModelID) - chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID) if err != nil { logger.Errorf(ctx, "Failed to get chat model: %v", err) return nil, nil, err } - logger.Info(ctx, "Setting up chat options") opt := &chat.ChatOptions{ Temperature: chatManage.SummaryConfig.Temperature, TopP: chatManage.SummaryConfig.TopP, diff --git a/internal/application/service/chat_pipline/extract_entity.go b/internal/application/service/chat_pipline/extract_entity.go index 677720f1..c23f451e 100644 --- a/internal/application/service/chat_pipline/extract_entity.go +++ b/internal/application/service/chat_pipline/extract_entity.go @@ -94,6 +94,7 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context, return next() } +// Extractor is a struct for extracting entities type Extractor struct { chat chat.Chat formater *Formater @@ -101,6 +102,7 @@ type Extractor struct { chatOpt *chat.ChatOptions } +// NewExtractor creates a new extractor func NewExtractor( chatModel chat.Chat, template *types.PromptTemplateStructured, @@ -118,6 +120,7 @@ func NewExtractor( } } +// Extract extracts entities from content func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) { generator := NewQAPromptGenerator(e.formater, e.template) @@ -139,6 +142,7 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa return graph, nil } +// RemoveUnknownRelation removes unknown relations from graph func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) { relationType := make(map[string]bool) for _, tag := range e.template.Tags { @@ -156,6 +160,7 @@ func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.Grap graph.Relation = relationNew } +// QAPromptGenerator is a struct for generating QA prompts type QAPromptGenerator struct { Formater *Formater Template *types.PromptTemplateStructured @@ -165,6 +170,7 @@ type QAPromptGenerator struct { AnswerPrefix string } +// NewQAPromptGenerator creates a new QA prompt generator func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator { return &QAPromptGenerator{ Formater: formater, @@ -176,6 +182,7 @@ func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStru } } +// System generates a system prompt func (qa *QAPromptGenerator) System(ctx context.Context) string { promptLines := []string{} @@ -205,6 +212,7 @@ func (qa *QAPromptGenerator) System(ctx context.Context) string { return strings.Join(promptLines, "\n") } +// User generates a user prompt func (qa *QAPromptGenerator) User(ctx context.Context, question string) string { promptLines := []string{} promptLines = append(promptLines, qa.QuestionHeading) @@ -213,6 +221,7 @@ func (qa *QAPromptGenerator) User(ctx context.Context, question string) string { return strings.Join(promptLines, "\n") } +// Render renders a prompt func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message { return []chat.Message{ { @@ -226,10 +235,13 @@ func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat } } +// FormatType is a type for format types type FormatType string const ( + // FormatTypeJSON is a format type for JSON FormatTypeJSON FormatType = "json" + // FormatTypeYAML is a format type for YAML FormatTypeYAML FormatType = "yaml" ) @@ -245,6 +257,7 @@ var _FENCE_RE = regexp.MustCompile( _FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END, ) +// Formater is a struct for formatting entities type Formater struct { attributeSuffix string formatType FormatType @@ -256,6 +269,7 @@ type Formater struct { relationPrefix string } +// NewFormater creates a new formater func NewFormater() *Formater { return &Formater{ attributeSuffix: "_attributes", @@ -268,6 +282,7 @@ func NewFormater() *Formater { } } +// formatExtraction formats extraction func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) { items := make([]map[string]interface{}, 0) for _, node := range nodes { diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index 3abd3fe5..868f37c0 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -51,6 +51,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context, "chunk_cnt": len(searchResult), }) + // If there are no search results, return early if len(searchResult) == 0 { pipelineWarn(ctx, "Merge", "output", map[string]interface{}{ "chunk_cnt": 0, @@ -212,6 +213,7 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ return nil } +// populateFAQAnswers populates FAQ answers for the search results func (p *PluginMerge) populateFAQAnswers( ctx context.Context, chatManage *types.ChatManage, @@ -300,6 +302,7 @@ func (p *PluginMerge) populateFAQAnswers( return results } +// buildFAQAnswerContent builds the content of a FAQ answer func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string { if meta == nil { return "" @@ -336,6 +339,7 @@ func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string { return strings.TrimSpace(builder.String()) } +// expandShortContextWithNeighbors expands the short context with neighbors func (p *PluginMerge) expandShortContextWithNeighbors( ctx context.Context, chatManage *types.ChatManage, @@ -580,10 +584,12 @@ func (p *PluginMerge) expandShortContextWithNeighbors( return results } +// runeLen returns the length of a string in runes func runeLen(s string) int { return len([]rune(s)) } +// mergeOrderedContent merges ordered content func mergeOrderedContent(prev, base, next string, maxLen int) string { content := base if prev != "" { @@ -599,7 +605,7 @@ func mergeOrderedContent(prev, base, next string, maxLen int) string { return content } -// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix +// concatNoOverlap concatenates two strings, removing potential overlapping prefix/suffix func concatNoOverlap(a, b string) string { if a == "" { return b diff --git a/internal/application/service/chat_pipline/preprocess.go b/internal/application/service/chat_pipline/preprocess.go index 029cc745..b6855bde 100644 --- a/internal/application/service/chat_pipline/preprocess.go +++ b/internal/application/service/chat_pipline/preprocess.go @@ -239,6 +239,7 @@ func isInformativeToken(token string) bool { return true } +// containsChineseCharacters checks if a string contains Chinese characters func containsChineseCharacters(text string) bool { for _, r := range text { if unicode.Is(unicode.Han, r) { @@ -248,6 +249,7 @@ func containsChineseCharacters(text string) bool { return false } +// containsLatinLetters checks if a string contains Latin letters func containsLatinLetters(text string) bool { for _, r := range text { if r <= unicode.MaxASCII && unicode.IsLetter(r) { @@ -257,22 +259,26 @@ func containsLatinLetters(text string) bool { return false } +// normalizeWhitespace normalizes whitespace in a string func normalizeWhitespace(text string) string { text = strings.TrimSpace(text) return multiSpaceRegex.ReplaceAllString(text, " ") } +// normalizeLatinQuery normalizes a Latin query func normalizeLatinQuery(text string) string { text = strings.ToLower(text) text = multiSpaceRegex.ReplaceAllString(text, " ") return strings.TrimSpace(text) } +// intentResp is a response for intent detection type intentResp struct { Intent string `json:"intent"` Confidence float64 `json:"confidence"` } +// detectIntentLLM detects the intent of a query using an LLM func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string { if p.modelService == nil || chatManage.ChatModelID == "" { pipelineWarn( @@ -330,6 +336,7 @@ func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *type } } +// extractJSONBody extracts a JSON body from a string func extractJSONBody(text string) string { t := strings.TrimSpace(text) // Try fenced block first diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 30df6896..ca7b487c 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -176,6 +176,7 @@ func (p *PluginRerank) rerank(ctx context.Context, return rankFilter } +// ensureMetadata ensures the metadata is not nil func ensureMetadata(m map[string]string) map[string]string { if m == nil { return make(map[string]string) @@ -183,6 +184,7 @@ func ensureMetadata(m map[string]string) map[string]string { return m } +// compositeScore calculates the composite score for a search result func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 { sourceWeight := 1.0 switch strings.ToLower(sr.KnowledgeSource) { @@ -222,6 +224,7 @@ func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatM return composite } +// applyMMR applies the MMR algorithm to the search results func applyMMR( ctx context.Context, results []*types.SearchResult, @@ -285,6 +288,7 @@ func applyMMR( return selected } +// tokenizeSimple tokenizes a text into a set of tokens func tokenizeSimple(text string) map[string]struct{} { text = strings.ToLower(text) fields := strings.Fields(text) @@ -297,6 +301,7 @@ func tokenizeSimple(text string) map[string]struct{} { return set } +// jaccard calculates the Jaccard similarity between two sets of tokens func jaccard(a, b map[string]struct{}) float64 { if len(a) == 0 && len(b) == 0 { return 0 @@ -314,6 +319,7 @@ func jaccard(a, b map[string]struct{}) float64 { return float64(inter) / float64(union) } +// clampFloat clamps a float value between a minimum and maximum value func clampFloat(v, minV, maxV float64) float64 { if v < minV { return minV diff --git a/internal/application/service/chat_pipline/search_entity.go b/internal/application/service/chat_pipline/search_entity.go index 62a2dbd4..24cf5276 100644 --- a/internal/application/service/chat_pipline/search_entity.go +++ b/internal/application/service/chat_pipline/search_entity.go @@ -16,6 +16,7 @@ type PluginSearchEntity struct { knowledgeRepo interfaces.KnowledgeRepository } +// NewPluginSearchEntity creates a new plugin search entity func NewPluginSearchEntity( eventManager *EventManager, graphRepository interfaces.RetrieveGraphRepository, @@ -31,12 +32,12 @@ func NewPluginSearchEntity( return res } -// ActivationEvents returns the event types this plugin handles +// ActivationEvents returns the list of event types this plugin responds to func (p *PluginSearchEntity) ActivationEvents() []types.EventType { return []types.EventType{types.ENTITY_SEARCH} } -// OnEvent handles search events in the chat pipeline +// OnEvent processes triggered events func (p *PluginSearchEntity) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { @@ -148,6 +149,7 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context, return next() } +// filterSeenChunk filters seen chunks from the graph func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string { seen := map[string]bool{} for _, chunk := range searchResult { @@ -169,6 +171,7 @@ func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult [ return chunkIDs } +// chunk2SearchResult converts a chunk to a search result func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult { return &types.SearchResult{ ID: chunk.ID, diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index 405a818f..a42c4a26 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -16,6 +16,7 @@ import ( "github.com/hibiken/asynq" ) +// NewChunkExtractTask creates a new chunk extract task func NewChunkExtractTask( ctx context.Context, client *asynq.Client, @@ -45,6 +46,7 @@ func NewChunkExtractTask( return nil } +// ChunkExtractService is a service for extracting chunks type ChunkExtractService struct { template *types.PromptTemplateStructured modelService interfaces.ModelService @@ -53,6 +55,7 @@ type ChunkExtractService struct { graphEngine interfaces.RetrieveGraphRepository } +// NewChunkExtractService creates a new chunk extract service func NewChunkExtractService( config *config.Config, modelService interfaces.ModelService, @@ -73,6 +76,7 @@ func NewChunkExtractService( } } +// Extract extracts a chunk func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error { var p types.ExtractChunkPayload if err := json.Unmarshal(t.Payload(), &p); err != nil { @@ -137,7 +141,5 @@ func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error logger.Errorf(ctx, "failed to add graph: %v", err) return err } - // gg, _ := json.Marshal(graph) - // logger.Infof(ctx, "extracted graph: %s", string(gg)) return nil } diff --git a/internal/application/service/mcp_service.go b/internal/application/service/mcp_service.go index 7bafe7d7..79899d00 100644 --- a/internal/application/service/mcp_service.go +++ b/internal/application/service/mcp_service.go @@ -193,12 +193,12 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types logger.GetLogger(ctx).Infof("MCP service disabled, connection closed: %s (ID: %s)", name, service.ID) } else if configChanged { s.mcpManager.CloseClient(service.ID) - logger.GetLogger(ctx).Infof("MCP service config changed, connection closed for reconnect: %s (ID: %s)", name, service.ID) + logger.GetLogger(ctx).Infof("MCP service config changed, connection closed: %s (ID: %s)", name, service.ID) } else if oldEnabled != existing.Enabled && existing.Enabled { // Service was just enabled (was disabled, now enabled) // Close any existing connection to ensure clean state s.mcpManager.CloseClient(service.ID) - logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed for clean state: %s (ID: %s)", name, service.ID) + logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed: %s (ID: %s)", name, service.ID) } logger.GetLogger(ctx).Infof("MCP service updated: %s (ID: %s), enabled: %v", name, service.ID, existing.Enabled) diff --git a/internal/application/service/session.go b/internal/application/service/session.go index 8c4da289..16ca79c9 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -634,7 +634,8 @@ func (s *sessionService) selectChatModelID( return session.SummaryModelID, nil } else if err == nil && model != nil { // Session has a model but it's not Remote, we'll check knowledge bases for Remote models - logger.Infof(ctx, "Session has summary model %s but it's not Remote, checking knowledge bases for Remote models", session.SummaryModelID) + logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+ + "checking knowledge bases for Remote models", session.SummaryModelID) } } @@ -952,7 +953,8 @@ func (s *sessionService) AgentQA( len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases) } } else { - logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v", len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases) + logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v", + len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases) } summaryModelID := session.SummaryModelID @@ -1141,7 +1143,8 @@ func (s *sessionService) SaveWebSearchTempKBState( } } -// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items +// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis +// and cleans up associated knowledge base and knowledge items. func (s *sessionService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error { if s.redisClient == nil { return nil diff --git a/internal/application/service/web_search/duckduckgo.go b/internal/application/service/web_search/duckduckgo.go index 296554bb..d9b59a6e 100644 --- a/internal/application/service/web_search/duckduckgo.go +++ b/internal/application/service/web_search/duckduckgo.go @@ -63,6 +63,7 @@ func (p *DuckDuckGoProvider) Search( return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr) } +// searchHTML performs a web search using DuckDuckGo HTML endpoint func (p *DuckDuckGoProvider) searchHTML( ctx context.Context, query string, @@ -134,6 +135,7 @@ func (p *DuckDuckGoProvider) searchHTML( return results, nil } +// searchAPI performs a web search using DuckDuckGo API endpoint func (p *DuckDuckGoProvider) searchAPI( ctx context.Context, query string, @@ -220,6 +222,7 @@ func (p *DuckDuckGoProvider) searchAPI( return results, nil } +// cleanDDGURL cleans the URL from DuckDuckGo HTML endpoint func cleanDDGURL(urlStr string) string { if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") { trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=") @@ -241,6 +244,7 @@ func cleanDDGURL(urlStr string) string { return urlStr } +// extractTitle extracts the title from the text func extractTitle(text string) string { lines := strings.Split(text, "\n") if len(lines) > 0 { diff --git a/internal/application/service/web_search/duckduckgo_test.go b/internal/application/service/web_search/duckduckgo_test.go index c1c8c539..3597762a 100644 --- a/internal/application/service/web_search/duckduckgo_test.go +++ b/internal/application/service/web_search/duckduckgo_test.go @@ -52,7 +52,7 @@ func TestDuckDuckGoProvider_Name(t *testing.T) { } } -func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) { +func TestDuckDuckGoProvider(t *testing.T) { // Minimal HTML page with two results, matching selectors used in searchHTML html := ` @@ -105,7 +105,7 @@ func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) { } } -func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) { +func TestDuckDuckGoProvider_Fallback(t *testing.T) { // Simulate HTML returning non-OK to force API fallback, then a minimal API JSON apiResp := struct { AbstractText string `json:"AbstractText"` @@ -233,7 +233,7 @@ func TestDuckDuckGoProvider_Search_Real(t *testing.T) { // TestDuckDuckGoProvider_Search_Real_Chinese tests the DuckDuckGo provider with Chinese query. // This verifies the Chinese language parameter (kl=cn-zh) works correctly. -func TestDuckDuckGoProvider_Search_Real_Chinese(t *testing.T) { +func TestDuckDuckGoProvider_SearchChinese(t *testing.T) { if testing.Short() { t.Skip("Skipping real DuckDuckGo integration test in short mode") } diff --git a/internal/handler/faq.go b/internal/handler/faq.go index bea127f6..e5c5d9a8 100644 --- a/internal/handler/faq.go +++ b/internal/handler/faq.go @@ -17,7 +17,7 @@ type FAQHandler struct { knowledgeService interfaces.KnowledgeService } -// NewFAQHandler creates a new FAQ handler. +// NewFAQHandler creates a new FAQ handler func NewFAQHandler(knowledgeService interfaces.KnowledgeService) *FAQHandler { return &FAQHandler{knowledgeService: knowledgeService} } @@ -132,14 +132,17 @@ func (h *FAQHandler) UpdateEntryStatusBatch(c *gin.Context) { }) } +// faqDeleteRequest is a request for deleting FAQ entries in batch type faqDeleteRequest struct { IDs []string `json:"ids" binding:"required,min=1,dive,required"` } +// faqEntryStatusBatchRequest is a request for updating the enable status of FAQ entries in batch type faqEntryStatusBatchRequest struct { Updates map[string]bool `json:"updates" binding:"required,min=1"` } +// faqEntryTagBatchRequest is a request for updating tags for FAQ entries in batch type faqEntryTagBatchRequest struct { Updates map[string]*string `json:"updates" binding:"required,min=1"` } diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 3c867631..d953b031 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -351,12 +351,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { ctx := c.Request.Context() kbIdStr := utils.SanitizeForLog(c.Param("kbId")) - var req InitializationRequest - if err := c.ShouldBindJSON(&req); err != nil { - logger.Error(ctx, "Failed to parse initialization request", err) - c.Error(errors.NewBadRequestError(err.Error())) + req, err := h.bindInitializationRequest(ctx, c) + if err != nil { + c.Error(err) return } + logger.Infof( ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", @@ -364,94 +364,152 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { utils.SanitizeForLog(utils.ToJSON(req)), ) - // 获取指定知识库信息 + kb, err := h.getKnowledgeBaseForInitialization(ctx, kbIdStr) + if err != nil { + c.Error(err) + return + } + + if err := h.validateInitializationConfigs(ctx, req); err != nil { + c.Error(err) + return + } + + processedModels, err := h.processInitializationModels(ctx, kb, kbIdStr, req) + if err != nil { + c.Error(err) + return + } + + h.applyKnowledgeBaseInitialization(kb, req, processedModels) + + if err := h.kbRepository.UpdateKnowledgeBase(ctx, kb); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) + c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error())) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "知识库配置更新成功", + "data": gin.H{ + "models": processedModels, + "knowledge_base": kb, + }, + }) +} + +func (h *InitializationHandler) bindInitializationRequest(ctx context.Context, c *gin.Context) (*InitializationRequest, error) { + var req InitializationRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse initialization request", err) + return nil, errors.NewBadRequestError(err.Error()) + } + return &req, nil +} + +func (h *InitializationHandler) getKnowledgeBaseForInitialization(ctx context.Context, kbIdStr string) (*types.KnowledgeBase, error) { kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) - c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error())) - return + return nil, errors.NewInternalServerError("获取知识库信息失败: " + err.Error()) } - if kb == nil { logger.Error(ctx, "Knowledge base not found") - c.Error(errors.NewNotFoundError("知识库不存在")) - return + return nil, errors.NewNotFoundError("知识库不存在") + } + return kb, nil +} + +func (h *InitializationHandler) validateInitializationConfigs(ctx context.Context, req *InitializationRequest) error { + if err := h.validateMultimodalConfig(ctx, req); err != nil { + return err + } + if err := validateRerankConfig(ctx, req); err != nil { + return err + } + return validateNodeExtractConfig(ctx, req) +} + +func (h *InitializationHandler) validateMultimodalConfig(ctx context.Context, req *InitializationRequest) error { + if !req.Multimodal.Enabled { + return nil } - // 验证多模态配置(如果启用) - if req.Multimodal.Enabled { - storageType := strings.ToLower(req.Multimodal.StorageType) - if req.Multimodal.VLM == nil { - logger.Error(ctx, "Multimodal enabled but missing VLM configuration") - c.Error(errors.NewBadRequestError("启用多模态时需要配置VLM信息")) - return - } - if req.Multimodal.VLM.InterfaceType == "ollama" { - req.Multimodal.VLM.BaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1" - } - if req.Multimodal.VLM.ModelName == "" || req.Multimodal.VLM.BaseURL == "" { - logger.Error(ctx, "VLM configuration incomplete") - c.Error(errors.NewBadRequestError("VLM配置不完整")) - return - } - switch storageType { - case "cos": - if req.Multimodal.COS == nil || req.Multimodal.COS.SecretID == "" || req.Multimodal.COS.SecretKey == "" || - req.Multimodal.COS.Region == "" || req.Multimodal.COS.BucketName == "" || - req.Multimodal.COS.AppID == "" { - logger.Error(ctx, "COS configuration incomplete") - c.Error(errors.NewBadRequestError("COS配置不完整")) - return - } - case "minio": - if req.Multimodal.Minio == nil || req.Multimodal.Minio.BucketName == "" || - os.Getenv("MINIO_ACCESS_KEY_ID") == "" || os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" { - logger.Error(ctx, "MinIO configuration incomplete") - c.Error(errors.NewBadRequestError("MinIO配置不完整")) - return - } - } + storageType := strings.ToLower(req.Multimodal.StorageType) + if req.Multimodal.VLM == nil { + logger.Error(ctx, "Multimodal enabled but missing VLM configuration") + return errors.NewBadRequestError("启用多模态时需要配置VLM信息") + } + if req.Multimodal.VLM.InterfaceType == "ollama" { + req.Multimodal.VLM.BaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1" + } + if req.Multimodal.VLM.ModelName == "" || req.Multimodal.VLM.BaseURL == "" { + logger.Error(ctx, "VLM configuration incomplete") + return errors.NewBadRequestError("VLM配置不完整") } - // 验证Rerank配置(如果启用) - if req.Rerank.Enabled { - if req.Rerank.ModelName == "" || req.Rerank.BaseURL == "" { - logger.Error(ctx, "Rerank configuration incomplete") - c.Error(errors.NewBadRequestError("Rerank配置不完整")) - return + switch storageType { + case "cos": + if req.Multimodal.COS == nil || req.Multimodal.COS.SecretID == "" || req.Multimodal.COS.SecretKey == "" || + req.Multimodal.COS.Region == "" || req.Multimodal.COS.BucketName == "" || + req.Multimodal.COS.AppID == "" { + logger.Error(ctx, "COS configuration incomplete") + return errors.NewBadRequestError("COS配置不完整") + } + case "minio": + if req.Multimodal.Minio == nil || req.Multimodal.Minio.BucketName == "" || + os.Getenv("MINIO_ACCESS_KEY_ID") == "" || os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" { + logger.Error(ctx, "MinIO configuration incomplete") + return errors.NewBadRequestError("MinIO配置不完整") } } + return nil +} - // 验证Node Extractor配置(如果启用) - if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" && req.NodeExtract.Enabled { +func validateRerankConfig(ctx context.Context, req *InitializationRequest) error { + if !req.Rerank.Enabled { + return nil + } + if req.Rerank.ModelName == "" || req.Rerank.BaseURL == "" { + logger.Error(ctx, "Rerank configuration incomplete") + return errors.NewBadRequestError("Rerank配置不完整") + } + return nil +} + +func validateNodeExtractConfig(ctx context.Context, req *InitializationRequest) error { + if !req.NodeExtract.Enabled { + return nil + } + if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" { logger.Error(ctx, "Node Extractor configuration incomplete") - c.Error(errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE")) - return + return errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE") } - if req.NodeExtract.Enabled { - if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 { - logger.Error(ctx, "Node Extractor configuration incomplete") - c.Error(errors.NewBadRequestError("Node Extractor配置不完整")) - return - } - if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 { - logger.Error(ctx, "Node Extractor configuration incomplete") - c.Error(errors.NewBadRequestError("请先提取实体和关系")) - return - } + if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 { + logger.Error(ctx, "Node Extractor configuration incomplete") + return errors.NewBadRequestError("Node Extractor配置不完整") } + if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 { + logger.Error(ctx, "Node Extractor configuration incomplete") + return errors.NewBadRequestError("请先提取实体和关系") + } + return nil +} - // 处理模型创建/更新 - modelsToProcess := []struct { - modelType types.ModelType - name string - source types.ModelSource - description string - baseURL string - apiKey string - dimension int - interfaceType string - }{ +type modelDescriptor struct { + modelType types.ModelType + name string + source types.ModelSource + description string + baseURL string + apiKey string + dimension int + interfaceType string +} + +func buildModelDescriptors(req *InitializationRequest) []modelDescriptor { + descriptors := []modelDescriptor{ { modelType: types.ModelTypeKnowledgeQA, name: utils.SanitizeForLog(req.LLM.ModelName), @@ -471,18 +529,8 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { }, } - // 如果启用Rerank,添加Rerank模型 if req.Rerank.Enabled { - modelsToProcess = append(modelsToProcess, struct { - modelType types.ModelType - name string - source types.ModelSource - description string - baseURL string - apiKey string - dimension int - interfaceType string - }{ + descriptors = append(descriptors, modelDescriptor{ modelType: types.ModelTypeRerank, name: utils.SanitizeForLog(req.Rerank.ModelName), source: types.ModelSourceRemote, @@ -492,18 +540,8 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { }) } - // 如果启用多模态,添加VLM模型 if req.Multimodal.Enabled && req.Multimodal.VLM != nil { - modelsToProcess = append(modelsToProcess, struct { - modelType types.ModelType - name string - source types.ModelSource - description string - baseURL string - apiKey string - dimension int - interfaceType string - }{ + descriptors = append(descriptors, modelDescriptor{ modelType: types.ModelTypeVLLM, name: utils.SanitizeForLog(req.Multimodal.VLM.ModelName), source: types.ModelSourceRemote, @@ -514,113 +552,116 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { }) } - // 处理模型 + return descriptors +} + +func (h *InitializationHandler) processInitializationModels( + ctx context.Context, + kb *types.KnowledgeBase, + kbIdStr string, + req *InitializationRequest, +) ([]*types.Model, error) { + descriptors := buildModelDescriptors(req) var processedModels []*types.Model - for _, modelInfo := range modelsToProcess { - model := &types.Model{ - Type: modelInfo.modelType, - Name: modelInfo.name, - Source: modelInfo.source, - Description: modelInfo.description, - Parameters: types.ModelParameters{ - BaseURL: modelInfo.baseURL, - APIKey: modelInfo.apiKey, - InterfaceType: modelInfo.interfaceType, - }, - IsDefault: false, - Status: types.ModelStatusActive, - } - if modelInfo.modelType == types.ModelTypeEmbedding { - model.Parameters.EmbeddingParameters = types.EmbeddingParameters{ - Dimension: modelInfo.dimension, - } - } + for _, descriptor := range descriptors { + model := descriptor.toModel() + existingModelID := h.findExistingModelID(kb, descriptor.modelType) - // 根据模型类型从知识库获取已有的模型ID - var existingModelID string - switch modelInfo.modelType { - case types.ModelTypeEmbedding: - existingModelID = kb.EmbeddingModelID - case types.ModelTypeKnowledgeQA: - existingModelID = kb.SummaryModelID - case types.ModelTypeVLLM: - existingModelID = kb.VLMConfig.ModelID - } - - // 如果知识库中已经有对应类型的模型ID,尝试获取并更新 var existingModel *types.Model if existingModelID != "" { + var err error existingModel, err = h.modelService.GetModelByID(ctx, existingModelID) if err != nil { - // 模型不存在或获取失败,记录日志但继续创建新模型 logger.Warnf(ctx, "Failed to get existing model %s: %v, will create new one", existingModelID, err) existingModel = nil } } if existingModel != nil { - // 更新现有模型 existingModel.Name = model.Name existingModel.Source = model.Source existingModel.Description = model.Description existingModel.Parameters = model.Parameters existingModel.UpdatedAt = time.Now() - err = h.modelService.UpdateModel(ctx, existingModel) - if err != nil { + if err := h.modelService.UpdateModel(ctx, existingModel); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "model_id": model.ID, "kb_id": kbIdStr, }) - c.Error(errors.NewInternalServerError("更新模型失败: " + err.Error())) - return + return nil, errors.NewInternalServerError("更新模型失败: " + err.Error()) } processedModels = append(processedModels, existingModel) - } else { - // 创建新模型 - err = h.modelService.CreateModel(ctx, model) - if err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{ - "model_id": model.ID, - "kb_id": kbIdStr, - }) - c.Error(errors.NewInternalServerError("创建模型失败: " + err.Error())) - return - } - processedModels = append(processedModels, model) - } - } - - // 找到模型ID - var embeddingModelID, llmModelID, vlmModelID string - for _, model := range processedModels { - if model == nil { continue } - if model.Type == types.ModelTypeEmbedding { - embeddingModelID = model.ID + + if err := h.modelService.CreateModel(ctx, model); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "kb_id": kbIdStr, + }) + return nil, errors.NewInternalServerError("创建模型失败: " + err.Error()) } - if model.Type == types.ModelTypeKnowledgeQA { - llmModelID = model.ID - } - if model.Type == types.ModelTypeVLLM { - vlmModelID = model.ID + processedModels = append(processedModels, model) + } + + return processedModels, nil +} + +func (descriptor modelDescriptor) toModel() *types.Model { + model := &types.Model{ + Type: descriptor.modelType, + Name: descriptor.name, + Source: descriptor.source, + Description: descriptor.description, + Parameters: types.ModelParameters{ + BaseURL: descriptor.baseURL, + APIKey: descriptor.apiKey, + InterfaceType: descriptor.interfaceType, + }, + IsDefault: false, + Status: types.ModelStatusActive, + } + + if descriptor.modelType == types.ModelTypeEmbedding { + model.Parameters.EmbeddingParameters = types.EmbeddingParameters{ + Dimension: descriptor.dimension, } } - // 更新知识库配置 + return model +} + +func (h *InitializationHandler) findExistingModelID(kb *types.KnowledgeBase, modelType types.ModelType) string { + switch modelType { + case types.ModelTypeEmbedding: + return kb.EmbeddingModelID + case types.ModelTypeKnowledgeQA: + return kb.SummaryModelID + case types.ModelTypeVLLM: + return kb.VLMConfig.ModelID + default: + return "" + } +} + +func (h *InitializationHandler) applyKnowledgeBaseInitialization( + kb *types.KnowledgeBase, + req *InitializationRequest, + processedModels []*types.Model, +) { + embeddingModelID, llmModelID, vlmModelID := extractModelIDs(processedModels) + kb.SummaryModelID = llmModelID kb.EmbeddingModelID = embeddingModelID - // 更新文档分割配置 kb.ChunkingConfig = types.ChunkingConfig{ ChunkSize: req.DocumentSplitting.ChunkSize, ChunkOverlap: req.DocumentSplitting.ChunkOverlap, Separators: req.DocumentSplitting.Separators, } - // 更新多模态配置 if req.Multimodal.Enabled { kb.VLMConfig = types.VLMConfig{ Enabled: req.Multimodal.Enabled, @@ -677,22 +718,23 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) { }) } } +} - err = h.kbRepository.UpdateKnowledgeBase(ctx, kb) - if err != nil { - logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)}) - c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error())) - return +func extractModelIDs(processedModels []*types.Model) (embeddingModelID, llmModelID, vlmModelID string) { + for _, model := range processedModels { + if model == nil { + continue + } + switch model.Type { + case types.ModelTypeEmbedding: + embeddingModelID = model.ID + case types.ModelTypeKnowledgeQA: + llmModelID = model.ID + case types.ModelTypeVLLM: + vlmModelID = model.ID + } } - - c.JSON(http.StatusOK, gin.H{ - "success": true, - "message": "知识库配置更新成功", - "data": gin.H{ - "models": processedModels, - "knowledge_base": kb, - }, - }) + return } // CheckOllamaStatus 检查Ollama服务状态 @@ -1787,6 +1829,7 @@ type TextRelationExtractionResponse struct { Relations []*types.GraphRelation `json:"relations"` } +// ExtractTextRelations extracts text relations from text func (h *InitializationHandler) ExtractTextRelations(c *gin.Context) { ctx := c.Request.Context() @@ -1869,15 +1912,18 @@ func (h *InitializationHandler) extractRelationsFromText( return result, nil } +// FabriTextRequest is a request for generating example text type FabriTextRequest struct { Tags []string `json:"tags"` LLMConfig LLMConfig `json:"llmConfig"` } +// FabriTextResponse is a response for generating example text type FabriTextResponse struct { Text string `json:"text"` } +// FabriText generates example text func (h *InitializationHandler) FabriText(c *gin.Context) { ctx := c.Request.Context() @@ -1901,6 +1947,7 @@ func (h *InitializationHandler) FabriText(c *gin.Context) { }) } +// fabriText generates example text func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, llm LLMConfig) (string, error) { chatModel, err := chat.NewChat(&chat.ChatConfig{ ModelID: "initialization", @@ -1935,10 +1982,12 @@ func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, ll return result.Content, nil } +// FabriTagRequest is a request for generating tags type FabriTagRequest struct { LLMConfig LLMConfig `json:"llmConfig"` } +// FabriTagResponse is a response for generating tags type FabriTagResponse struct { Tags []string `json:"tags"` } @@ -1947,6 +1996,7 @@ var tagOptions = []string{ "内容", "文化", "人物", "事件", "时间", "地点", "作品", "作者", "关系", "属性", } +// FabriTag generates tags func (h *InitializationHandler) FabriTag(c *gin.Context) { tagRandom := RandomSelect(tagOptions, rand.Intn(len(tagOptions)-1)+1) c.JSON(http.StatusOK, gin.H{ @@ -1955,6 +2005,7 @@ func (h *InitializationHandler) FabriTag(c *gin.Context) { }) } +// RandomSelect selects random strings func RandomSelect(strs []string, n int) []string { if n <= 0 { return []string{} diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index a6f4db25..332456bc 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -159,7 +159,7 @@ func (h *Handler) AgentQA(c *gin.Context) { if requestJSON, err := json.Marshal(request); err == nil { logger.Infof(ctx, "Agent QA request, request: %s", secutils.SanitizeForLog(string(requestJSON))) } else { - logger.Warnf(ctx, "Agent QA request received but failed to marshal for logging: %s", secutils.SanitizeForLog(err.Error())) + logger.Warnf(ctx, "failed to marshal for logging: %s", secutils.SanitizeForLog(err.Error())) } // Validate query content diff --git a/internal/handler/tenant.go b/internal/handler/tenant.go index 7999d337..243d2e57 100644 --- a/internal/handler/tenant.go +++ b/internal/handler/tenant.go @@ -431,7 +431,7 @@ func (h *TenantHandler) UpdateTenantKV(c *gin.Context) { h.updateTenantWebSearchConfigInternal(c) return case "conversation-config": - h.updateTenantConversationConfigInternal(c) + h.updateTenantConversationInternal(c) return default: logger.Info(ctx, "KV key not supported", "key", key) @@ -664,9 +664,9 @@ func (h *TenantHandler) GetTenantConversationConfig(c *gin.Context) { }) } -// updateTenantConversationConfigInternal updates the conversation configuration for a tenant +// updateTenantConversationInternal updates the conversation configuration for a tenant // This sets the global conversation configuration for normal mode sessions in this tenant -func (h *TenantHandler) updateTenantConversationConfigInternal(c *gin.Context) { +func (h *TenantHandler) updateTenantConversationInternal(c *gin.Context) { ctx := c.Request.Context() logger.Info(ctx, "Start updating tenant conversation config") diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index dbcd6e6f..018ca3e0 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -43,7 +43,8 @@ type ChunkRepository interface { CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error) // DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range DeleteUnindexedChunks(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) - // ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID (only ID and ContentHash fields for efficiency) + // ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID + // only ID and ContentHash fields for efficiency ListAllFAQChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) } diff --git a/internal/types/interfaces/extracter.go b/internal/types/interfaces/extracter.go index c764ad15..7dc81e1a 100644 --- a/internal/types/interfaces/extracter.go +++ b/internal/types/interfaces/extracter.go @@ -6,6 +6,8 @@ import ( "github.com/hibiken/asynq" ) +// Extracter is a interface for extracting entities type Extracter interface { + // Extract extracts entities from a task Extract(ctx context.Context, t *asynq.Task) error } diff --git a/internal/types/interfaces/model.go b/internal/types/interfaces/model.go index efcd5ec2..987a2ae3 100644 --- a/internal/types/interfaces/model.go +++ b/internal/types/interfaces/model.go @@ -46,6 +46,7 @@ type ModelRepository interface { Update(ctx context.Context, model *types.Model) error // Delete deletes a model Delete(ctx context.Context, tenantID uint64, id string) error - // ClearDefaultByType clears the default flag for all models of a specific type, optionally excluding a specific model ID + // ClearDefaultByType clears the default flag for all models of a specific type + // optionally excluding a specific model ID. ClearDefaultByType(ctx context.Context, tenantID uint, modelType types.ModelType, excludeID string) error } diff --git a/internal/types/interfaces/retriever_graph.go b/internal/types/interfaces/retriever_graph.go index 04f939d6..9b6333ce 100644 --- a/internal/types/interfaces/retriever_graph.go +++ b/internal/types/interfaces/retriever_graph.go @@ -6,8 +6,12 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +// RetrieveGraphRepository is a repository for retrieving graphs type RetrieveGraphRepository interface { + // AddGraph adds a graph to the repository AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error + // DelGraph deletes a graph from the repository DelGraph(ctx context.Context, namespace []types.NameSpace) error + // SearchNode searches for nodes in the repository SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) } diff --git a/internal/types/interfaces/session.go b/internal/types/interfaces/session.go index b0e0b6bc..e344e614 100644 --- a/internal/types/interfaces/session.go +++ b/internal/types/interfaces/session.go @@ -32,7 +32,8 @@ type SessionService interface { // webSearchEnabled: whether to enable web search to supplement knowledge base results // Events are emitted through eventBus (references, answer chunks, completion) KnowledgeQA(ctx context.Context, - session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus, + session *types.Session, query string, knowledgeBaseIDs []string, + assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus, ) error // KnowledgeQAByEvent performs knowledge-based question answering by event KnowledgeQAByEvent(ctx context.Context, chatManage *types.ChatManage, eventList []types.EventType) error @@ -62,7 +63,8 @@ type SessionService interface { seenURLs map[string]bool, knowledgeIDs []string, ) - // DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items + // DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis + // and cleans up associated knowledge base and knowledge items DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error } diff --git a/internal/types/interfaces/web_search.go b/internal/types/interfaces/web_search.go index a7814a7a..9e2fddbe 100644 --- a/internal/types/interfaces/web_search.go +++ b/internal/types/interfaces/web_search.go @@ -8,12 +8,18 @@ import ( // WebSearchProvider defines the interface for web search providers type WebSearchProvider interface { - Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) + // Name returns the name of the provider Name() string + // Search performs a web search + Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) } +// WebSearchService defines the interface for web search services type WebSearchService interface { + // Search performs a web search Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) + // CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base + // The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering. CompressWithRAG(ctx context.Context, sessionID string, tempKBID string, questions []string, webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig, kbSvc KnowledgeBaseService, knowSvc KnowledgeService, diff --git a/internal/utils/json.go b/internal/utils/json.go index 0070b469..b7fe5315 100644 --- a/internal/utils/json.go +++ b/internal/utils/json.go @@ -2,6 +2,7 @@ package utils import "encoding/json" +// ToJSON converts a value to a JSON string func ToJSON(v interface{}) string { json, err := json.Marshal(v) if err != nil {