refactor: Remove agent_test command-line tool and associated files to streamline project structure and eliminate unused components
This commit is contained in:
@@ -1,75 +0,0 @@
|
||||
.PHONY: build run clean help
|
||||
|
||||
# Binary name
|
||||
BINARY=agent_test
|
||||
|
||||
# Build parameters
|
||||
GOOS?=$(shell go env GOOS)
|
||||
GOARCH?=$(shell go env GOARCH)
|
||||
|
||||
# Default target
|
||||
all: build
|
||||
|
||||
# Build the binary
|
||||
build:
|
||||
@echo "Building $(BINARY)..."
|
||||
@go build -o $(BINARY) .
|
||||
@echo "Build complete: ./$(BINARY)"
|
||||
|
||||
# Build and run with default settings
|
||||
run: build
|
||||
@./$(BINARY)
|
||||
|
||||
# Run with custom parameters
|
||||
run-local: build
|
||||
@./$(BINARY) -url http://localhost:8080 -kb $(KB_ID)
|
||||
|
||||
# Run with existing session
|
||||
run-session: build
|
||||
@./$(BINARY) -url http://localhost:8080 -session $(SESSION_ID)
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
@echo "Cleaning..."
|
||||
@rm -f $(BINARY)
|
||||
@echo "Clean complete"
|
||||
|
||||
# Install dependencies
|
||||
deps:
|
||||
@echo "Installing dependencies..."
|
||||
@go mod download
|
||||
@go mod tidy
|
||||
@echo "Dependencies installed"
|
||||
|
||||
# Run tests
|
||||
test:
|
||||
@go test -v ./...
|
||||
|
||||
# Build for multiple platforms
|
||||
build-all:
|
||||
@echo "Building for multiple platforms..."
|
||||
@GOOS=linux GOARCH=amd64 go build -o $(BINARY)-linux-amd64 .
|
||||
@GOOS=darwin GOARCH=amd64 go build -o $(BINARY)-darwin-amd64 .
|
||||
@GOOS=darwin GOARCH=arm64 go build -o $(BINARY)-darwin-arm64 .
|
||||
@GOOS=windows GOARCH=amd64 go build -o $(BINARY)-windows-amd64.exe .
|
||||
@echo "Build complete for all platforms"
|
||||
|
||||
# Help
|
||||
help:
|
||||
@echo "Available targets:"
|
||||
@echo " make build - Build the binary"
|
||||
@echo " make run - Build and run with no parameters"
|
||||
@echo " make run-local - Build and run with local server (requires KB_ID env var)"
|
||||
@echo " make run-session - Build and run with existing session (requires SESSION_ID env var)"
|
||||
@echo " make clean - Remove build artifacts"
|
||||
@echo " make deps - Install dependencies"
|
||||
@echo " make test - Run tests"
|
||||
@echo " make build-all - Build for multiple platforms"
|
||||
@echo " make help - Show this help message"
|
||||
@echo ""
|
||||
@echo "Examples:"
|
||||
@echo " make build"
|
||||
@echo " KB_ID=kb_123 make run-local"
|
||||
@echo " SESSION_ID=sess_456 make run-session"
|
||||
@echo " ./$(BINARY) -url http://localhost:8080 -kb kb_123 -token mytoken"
|
||||
|
||||
@@ -1,293 +0,0 @@
|
||||
# Quick Start Guide
|
||||
|
||||
Get started with the WeKnora Agent Test Tool in 5 minutes!
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Go 1.21 or higher
|
||||
- Access to a running WeKnora instance
|
||||
- A knowledge base ID (or create one through the tool)
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Option 1: Using the Build Script (Recommended)
|
||||
|
||||
```bash
|
||||
# 1. Navigate to the tool directory
|
||||
cd client/cmd/agent_test
|
||||
|
||||
# 2. Build and run with your knowledge base
|
||||
./run.sh -k kb_your_knowledge_base_id
|
||||
|
||||
# Or with a remote server
|
||||
./run.sh -u https://api.weknora.com -k kb_12345 -t your_token
|
||||
```
|
||||
|
||||
### Option 2: Using Make
|
||||
|
||||
```bash
|
||||
# Build the binary
|
||||
make build
|
||||
|
||||
# Run with environment variables
|
||||
KB_ID=kb_12345 make run-local
|
||||
|
||||
# Or run the binary directly
|
||||
./agent_test -url http://localhost:8080 -kb kb_12345
|
||||
```
|
||||
|
||||
### Option 3: Direct Go Build
|
||||
|
||||
```bash
|
||||
# Build
|
||||
go build -o agent_test .
|
||||
|
||||
# Run
|
||||
./agent_test -url http://localhost:8080 -kb kb_your_knowledge_base_id
|
||||
```
|
||||
|
||||
## First Steps
|
||||
|
||||
Once the tool is running, you'll see an interactive prompt:
|
||||
|
||||
```
|
||||
╔════════════════════════════════════════════════════════════╗
|
||||
║ WeKnora Agent QA Testing Tool ║
|
||||
╚════════════════════════════════════════════════════════════╝
|
||||
|
||||
[No Session] >
|
||||
```
|
||||
|
||||
### 1. Create a New Session
|
||||
|
||||
```
|
||||
> new
|
||||
```
|
||||
|
||||
The tool will create a new session and show you the session ID.
|
||||
|
||||
### 2. Ask Your First Question
|
||||
|
||||
```
|
||||
[Session: abc12345] > ask What is machine learning?
|
||||
```
|
||||
|
||||
You'll see real-time output showing:
|
||||
- 💭 Agent thinking process
|
||||
- 🔧 Tool calls (if any)
|
||||
- 📚 Knowledge references
|
||||
- 📝 Final answer
|
||||
- 🤔 Reflections (if any)
|
||||
|
||||
### 3. Continue the Conversation
|
||||
|
||||
```
|
||||
[Session: abc12345] > ask Can you give me examples?
|
||||
[Session: abc12345] > ask How does it work?
|
||||
```
|
||||
|
||||
### 4. View Session Info
|
||||
|
||||
```
|
||||
[Session: abc12345] > info
|
||||
```
|
||||
|
||||
### 5. List All Sessions
|
||||
|
||||
```
|
||||
[Session: abc12345] > sessions
|
||||
```
|
||||
|
||||
### 6. Exit
|
||||
|
||||
```
|
||||
[Session: abc12345] > exit
|
||||
```
|
||||
|
||||
## Command Reference
|
||||
|
||||
| Command | Description | Example |
|
||||
|---------|-------------|---------|
|
||||
| `new` | Create a new session | `> new` |
|
||||
| `ask <query>` | Ask the agent a question | `> ask What is AI?` |
|
||||
| `info` | Show current session details | `> info` |
|
||||
| `sessions` | List all your sessions | `> sessions` |
|
||||
| `switch <id>` | Switch to another session | `> switch sess_123` |
|
||||
| `help` | Show help message | `> help` |
|
||||
| `exit` or `quit` | Exit the program | `> exit` |
|
||||
|
||||
## Example Session
|
||||
|
||||
```bash
|
||||
$ ./run.sh -k kb_ml_docs
|
||||
|
||||
[No Session] > new
|
||||
✓ Session created successfully!
|
||||
Session ID: sess_a1b2c3d4
|
||||
Knowledge Base: kb_ml_docs
|
||||
Created At: 2025-11-03T10:00:00Z
|
||||
|
||||
[Session: sess_a1b] > ask What is a neural network?
|
||||
|
||||
╔═══ Agent Processing ═══╗
|
||||
Query: What is a neural network?
|
||||
|
||||
💭 Thinking: I need to search the knowledge base for information about neural networks...
|
||||
|
||||
🔧 Tool Call: search_knowledge
|
||||
Arguments: map[query:neural network definition]
|
||||
|
||||
✓ Tool Result:
|
||||
Found 3 relevant knowledge chunks
|
||||
|
||||
📚 Knowledge References: Found 3 reference(s)
|
||||
1. [Score: 0.892] A neural network is a series of algorithms...
|
||||
Knowledge: Neural Networks 101 (Chunk: 2)
|
||||
2. [Score: 0.856] Neural networks mimic biological neurons...
|
||||
Knowledge: Deep Learning Basics (Chunk: 5)
|
||||
3. [Score: 0.834] The structure consists of layers...
|
||||
Knowledge: ML Architectures (Chunk: 8)
|
||||
|
||||
📝 Final Answer:
|
||||
A neural network is a computing system inspired by biological neural networks
|
||||
in animal brains. It consists of interconnected nodes (artificial neurons)
|
||||
organized in layers that process information using a connectionist approach
|
||||
to computation. Neural networks learn to perform tasks by considering examples,
|
||||
generally without being programmed with task-specific rules.
|
||||
|
||||
🤔 Reflection: I successfully found relevant information in the knowledge base
|
||||
and provided a comprehensive answer covering the definition, structure, and
|
||||
learning mechanism of neural networks.
|
||||
|
||||
╔═══ Summary ═══╗
|
||||
Duration: 2.34s
|
||||
Tool Calls: 1 (search_knowledge)
|
||||
References: 3
|
||||
Reflections: 1
|
||||
╚═══════════════╝
|
||||
|
||||
[Session: sess_a1b] > ask What are the main types?
|
||||
|
||||
╔═══ Agent Processing ═══╗
|
||||
Query: What are the main types?
|
||||
|
||||
💭 Thinking: The user is asking about types of neural networks based on the
|
||||
previous context...
|
||||
|
||||
[... continuing the conversation ...]
|
||||
```
|
||||
|
||||
## Testing Scenarios
|
||||
|
||||
We provide a test scenarios script for common testing patterns:
|
||||
|
||||
```bash
|
||||
# Run interactive scenario menu
|
||||
./test_scenarios.sh
|
||||
|
||||
# Run a specific scenario
|
||||
./test_scenarios.sh 1 # Basic Q&A
|
||||
./test_scenarios.sh 2 # Complex queries
|
||||
./test_scenarios.sh 3 # Multi-turn conversation
|
||||
```
|
||||
|
||||
## Configuration with Environment Variables
|
||||
|
||||
Create a configuration file for repeated use:
|
||||
|
||||
```bash
|
||||
# Create config file
|
||||
cat > config.env << EOF
|
||||
export WEKNORA_URL=http://localhost:8080
|
||||
export WEKNORA_KB_ID=kb_your_knowledge_base_id
|
||||
export WEKNORA_TOKEN=your_token_if_needed
|
||||
EOF
|
||||
|
||||
# Load and run
|
||||
source config.env
|
||||
./run.sh
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Connection refused"
|
||||
- Make sure WeKnora server is running
|
||||
- Check the URL is correct
|
||||
- Verify network connectivity
|
||||
|
||||
### "Knowledge base not found"
|
||||
- Verify your knowledge base ID is correct
|
||||
- Check you have access to the knowledge base
|
||||
- Ensure the knowledge base has been initialized
|
||||
|
||||
### "Session not found"
|
||||
- The session may have expired or been deleted
|
||||
- Create a new session with `new` command
|
||||
|
||||
### No tool calls happening
|
||||
- Check your knowledge base has content
|
||||
- Verify the agent configuration includes tool definitions
|
||||
- Look for error messages in the output
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Using with Docker
|
||||
|
||||
If WeKnora is running in Docker:
|
||||
|
||||
```bash
|
||||
# Get the container's IP or use host networking
|
||||
./agent_test -url http://localhost:8080 -kb kb_123
|
||||
```
|
||||
|
||||
### Testing with Production
|
||||
|
||||
```bash
|
||||
# Use production URL with authentication
|
||||
./agent_test \
|
||||
-url https://api.weknora.com \
|
||||
-token $PROD_TOKEN \
|
||||
-kb kb_prod_123
|
||||
```
|
||||
|
||||
### Continuing an Existing Session
|
||||
|
||||
```bash
|
||||
# Skip session creation and continue existing one
|
||||
./agent_test -url http://localhost:8080 -session sess_existing_id
|
||||
```
|
||||
|
||||
### Batch Testing
|
||||
|
||||
```bash
|
||||
# Pipe commands for automated testing
|
||||
echo -e "new\nask What is AI?\ninfo\nexit" | ./agent_test -url http://localhost:8080 -kb kb_123
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Read the full [README.md](README.md) for detailed documentation
|
||||
- Check [agent_example.go](../../agent_example.go) for code examples
|
||||
- Explore the [client package documentation](../../README.md)
|
||||
- Review agent events and response types in the main README
|
||||
|
||||
## Getting Help
|
||||
|
||||
If you encounter issues:
|
||||
|
||||
1. Check the [README.md](README.md) for detailed documentation
|
||||
2. Review the error messages - they usually indicate the problem
|
||||
3. Verify your configuration (URL, KB ID, token)
|
||||
4. Check WeKnora server logs for backend issues
|
||||
5. Use `info` command to check session state
|
||||
|
||||
## Tips
|
||||
|
||||
- **Start Simple**: Begin with basic questions to verify connectivity
|
||||
- **Use Info**: The `info` command shows valuable session configuration
|
||||
- **Watch Events**: Pay attention to all event types to understand agent behavior
|
||||
- **Check References**: Knowledge references show if retrieval is working correctly
|
||||
- **Monitor Performance**: The summary shows timing and tool usage statistics
|
||||
|
||||
Happy testing! 🚀
|
||||
|
||||
@@ -1,307 +0,0 @@
|
||||
# WeKnora Agent QA Testing Tool
|
||||
|
||||
An interactive command-line tool for testing and debugging WeKnora's Agent QA functionality.
|
||||
|
||||
## Features
|
||||
|
||||
- 🎯 Interactive terminal interface with colored output
|
||||
- 📊 Real-time streaming of agent events
|
||||
- 🔧 Display tool calls and their results
|
||||
- 💭 Show agent thinking process
|
||||
- 📚 Display knowledge references with scores
|
||||
- 🤔 Capture agent reflections
|
||||
- 📝 Track complete conversation flow
|
||||
- ⏱️ Performance metrics and timing
|
||||
- 🔄 Session management (create, switch, list)
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cd client/cmd/agent_test
|
||||
go build -o agent_test
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Start with a new session
|
||||
./agent_test -url http://localhost:8080 -kb <knowledge_base_id>
|
||||
|
||||
# With authentication token
|
||||
./agent_test -url http://localhost:8080 -token <your_token> -kb <knowledge_base_id>
|
||||
|
||||
# Use existing session
|
||||
./agent_test -url http://localhost:8080 -session <session_id>
|
||||
```
|
||||
|
||||
### Command Line Flags
|
||||
|
||||
- `-url` : WeKnora API base URL (default: http://localhost:8080)
|
||||
- `-token` : API authentication token (optional)
|
||||
- `-kb` : Knowledge base ID (required for creating new sessions)
|
||||
- `-session` : Existing session ID to continue conversation (optional)
|
||||
|
||||
### Interactive Commands
|
||||
|
||||
Once the tool is running, you can use these commands:
|
||||
|
||||
#### Session Management
|
||||
|
||||
- `new` - Create a new session
|
||||
```
|
||||
> new
|
||||
```
|
||||
|
||||
- `sessions` - List all sessions
|
||||
```
|
||||
> sessions
|
||||
```
|
||||
|
||||
- `switch <session_id>` - Switch to another session
|
||||
```
|
||||
> switch abc123...
|
||||
```
|
||||
|
||||
- `info` - Show current session information
|
||||
```
|
||||
> info
|
||||
```
|
||||
|
||||
#### Agent Interaction
|
||||
|
||||
- `ask <query>` - Ask the agent a question
|
||||
```
|
||||
> ask What is machine learning?
|
||||
> ask How can I improve model accuracy?
|
||||
> ask Explain the difference between supervised and unsupervised learning
|
||||
```
|
||||
|
||||
#### General
|
||||
|
||||
- `help` - Show help message
|
||||
- `exit` or `quit` - Exit the program
|
||||
|
||||
## Agent Events
|
||||
|
||||
The tool displays various agent events in real-time:
|
||||
|
||||
### 💭 Thinking Events
|
||||
Shows the agent's internal reasoning process
|
||||
```
|
||||
💭 Thinking: Let me search for information about machine learning...
|
||||
```
|
||||
|
||||
### 🔧 Tool Call Events
|
||||
Displays when the agent calls a tool
|
||||
```
|
||||
🔧 Tool Call: search_knowledge
|
||||
Arguments: {"query": "machine learning definition"}
|
||||
```
|
||||
|
||||
### ✓ Tool Result Events
|
||||
Shows the results from tool execution
|
||||
```
|
||||
✓ Tool Result:
|
||||
Found 5 relevant knowledge chunks
|
||||
```
|
||||
|
||||
### 📚 Knowledge References
|
||||
Lists retrieved knowledge with relevance scores
|
||||
```
|
||||
📚 Knowledge References: Found 3 reference(s)
|
||||
1. [Score: 0.875] Machine learning is a subset of artificial intelligence...
|
||||
Knowledge: ML Basics (Chunk: 2)
|
||||
2. [Score: 0.823] There are three main types of machine learning...
|
||||
Knowledge: ML Types (Chunk: 5)
|
||||
```
|
||||
|
||||
### 📝 Final Answer
|
||||
The agent's complete response
|
||||
```
|
||||
📝 Final Answer:
|
||||
Machine learning is an application of artificial intelligence that provides
|
||||
systems the ability to automatically learn and improve from experience...
|
||||
```
|
||||
|
||||
### 🤔 Reflection Events
|
||||
Agent's self-reflection on its responses
|
||||
```
|
||||
🤔 Reflection: I provided a comprehensive answer covering the main aspects
|
||||
of machine learning with relevant examples.
|
||||
```
|
||||
|
||||
### Summary
|
||||
At the end of each query, a summary is displayed:
|
||||
```
|
||||
╔═══ Summary ═══╗
|
||||
Duration: 3.45s
|
||||
Tool Calls: 2 (search_knowledge, format_response)
|
||||
References: 3
|
||||
Reflections: 1
|
||||
╚═══════════════╝
|
||||
```
|
||||
|
||||
## Example Session
|
||||
|
||||
```bash
|
||||
$ ./agent_test -url http://localhost:8080 -kb kb_12345
|
||||
|
||||
╔════════════════════════════════════════════════════════════╗
|
||||
║ WeKnora Agent QA Testing Tool ║
|
||||
╚════════════════════════════════════════════════════════════╝
|
||||
|
||||
[No Session] > new
|
||||
✓ Session created successfully!
|
||||
Session ID: sess_abc123xyz
|
||||
Knowledge Base: kb_12345
|
||||
Created At: 2025-11-03T10:30:00Z
|
||||
|
||||
[Session: sess_abc] > ask What is neural networks?
|
||||
|
||||
╔═══ Agent Processing ═══╗
|
||||
Query: What is neural networks?
|
||||
|
||||
💭 Thinking: I need to search for information about neural networks...
|
||||
|
||||
🔧 Tool Call: search_knowledge
|
||||
Arguments: map[query:neural networks]
|
||||
|
||||
✓ Tool Result:
|
||||
Successfully retrieved 3 relevant chunks
|
||||
|
||||
📚 Knowledge References: Found 3 reference(s)
|
||||
1. [Score: 0.892] A neural network is a series of algorithms...
|
||||
Knowledge: Deep Learning Fundamentals (Chunk: 1)
|
||||
2. [Score: 0.856] Neural networks consist of layers of neurons...
|
||||
Knowledge: Neural Network Architecture (Chunk: 3)
|
||||
|
||||
📝 Final Answer:
|
||||
Neural networks are computing systems inspired by biological neural networks
|
||||
that constitute animal brains. They consist of interconnected nodes (neurons)
|
||||
organized in layers...
|
||||
|
||||
🤔 Reflection: The answer provides a clear definition with supporting
|
||||
references from the knowledge base.
|
||||
|
||||
╔═══ Summary ═══╗
|
||||
Duration: 2.87s
|
||||
Tool Calls: 1 (search_knowledge)
|
||||
References: 3
|
||||
Reflections: 1
|
||||
╚═══════════════╝
|
||||
|
||||
[Session: sess_abc] > info
|
||||
|
||||
╔═══ Session Information ═══╗
|
||||
Session ID: sess_abc123xyz
|
||||
Knowledge Base ID: kb_12345
|
||||
Max Rounds: 10
|
||||
Created At: 2025-11-03T10:30:00Z
|
||||
╚═══════════════════════════╝
|
||||
|
||||
[Session: sess_abc] > exit
|
||||
|
||||
Goodbye!
|
||||
```
|
||||
|
||||
## Testing Scenarios
|
||||
|
||||
### 1. Basic Q&A
|
||||
Test simple question-answering with knowledge retrieval:
|
||||
```
|
||||
> ask What is machine learning?
|
||||
```
|
||||
|
||||
### 2. Multi-Turn Conversation
|
||||
Test context awareness:
|
||||
```
|
||||
> ask What is supervised learning?
|
||||
> ask Can you give me some examples?
|
||||
> ask How does it differ from unsupervised learning?
|
||||
```
|
||||
|
||||
### 3. Complex Queries
|
||||
Test agent's reasoning capabilities:
|
||||
```
|
||||
> ask Compare and contrast supervised learning and reinforcement learning, including their use cases
|
||||
```
|
||||
|
||||
### 4. Tool Usage
|
||||
Test tool calling and reflection:
|
||||
```
|
||||
> ask Search for information about deep learning and summarize the key concepts
|
||||
```
|
||||
|
||||
### 5. Edge Cases
|
||||
Test error handling:
|
||||
```
|
||||
> ask [empty query]
|
||||
> ask [very long query with 1000+ characters]
|
||||
```
|
||||
|
||||
## Debugging
|
||||
|
||||
The tool displays detailed information for debugging:
|
||||
|
||||
1. **Event Timing**: Track how long each phase takes
|
||||
2. **Tool Arguments**: See exactly what parameters are passed to tools
|
||||
3. **Knowledge Scores**: Evaluate retrieval quality
|
||||
4. **Error Messages**: Clear error reporting for failed operations
|
||||
5. **Session State**: Monitor session configuration and status
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Connection Issues
|
||||
```
|
||||
✗ Request failed: connection refused
|
||||
```
|
||||
- Check if WeKnora server is running
|
||||
- Verify the URL is correct
|
||||
- Check network connectivity
|
||||
|
||||
### Authentication Errors
|
||||
```
|
||||
✗ HTTP error 401: Unauthorized
|
||||
```
|
||||
- Verify your API token is valid
|
||||
- Check token hasn't expired
|
||||
|
||||
### Session Not Found
|
||||
```
|
||||
✗ Failed to get session info: session not found
|
||||
```
|
||||
- Session may have been deleted
|
||||
- Create a new session with `new` command
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Using with Different Environments
|
||||
|
||||
Development:
|
||||
```bash
|
||||
./agent_test -url http://localhost:8080 -kb kb_dev
|
||||
```
|
||||
|
||||
Staging:
|
||||
```bash
|
||||
./agent_test -url https://staging.weknora.com -token $STAGING_TOKEN -kb kb_staging
|
||||
```
|
||||
|
||||
Production:
|
||||
```bash
|
||||
./agent_test -url https://api.weknora.com -token $PROD_TOKEN -kb kb_prod
|
||||
```
|
||||
|
||||
### Scripting
|
||||
|
||||
You can pipe commands for automated testing:
|
||||
```bash
|
||||
echo -e "new\nask What is AI?\nexit" | ./agent_test -url http://localhost:8080 -kb kb_test
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
Copyright (c) 2025 Tencent. All rights reserved.
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
module github.com/Tencent/WeKnora/client/cmd/agent_test
|
||||
|
||||
go 1.24.2
|
||||
|
||||
require github.com/Tencent/WeKnora/client v0.0.0
|
||||
|
||||
replace github.com/Tencent/WeKnora/client => ../..
|
||||
@@ -1,373 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/client"
|
||||
)
|
||||
|
||||
// Colors for terminal output
|
||||
const (
|
||||
ColorReset = "\033[0m"
|
||||
ColorRed = "\033[31m"
|
||||
ColorGreen = "\033[32m"
|
||||
ColorYellow = "\033[33m"
|
||||
ColorBlue = "\033[34m"
|
||||
ColorMagenta = "\033[35m"
|
||||
ColorCyan = "\033[36m"
|
||||
ColorWhite = "\033[37m"
|
||||
ColorBold = "\033[1m"
|
||||
)
|
||||
|
||||
// Config holds the configuration
|
||||
type Config struct {
|
||||
BaseURL string
|
||||
Token string
|
||||
KnowledgeBaseID string
|
||||
}
|
||||
|
||||
// CLI represents the command line interface
|
||||
type CLI struct {
|
||||
client *client.Client
|
||||
config *Config
|
||||
currentSessionID string
|
||||
agentSession *client.AgentSession
|
||||
}
|
||||
|
||||
func main() {
|
||||
baseURL := flag.String("url", "http://localhost:8080", "WeKnora API base URL")
|
||||
token := flag.String("token", "", "API authentication token")
|
||||
kbID := flag.String("kb", "", "Knowledge base ID")
|
||||
sessionID := flag.String("session", "", "Existing session ID (optional)")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
config := &Config{
|
||||
BaseURL: *baseURL,
|
||||
Token: *token,
|
||||
KnowledgeBaseID: *kbID,
|
||||
}
|
||||
|
||||
cli := NewCLI(config)
|
||||
|
||||
// Use existing session if provided
|
||||
if *sessionID != "" {
|
||||
cli.currentSessionID = *sessionID
|
||||
cli.agentSession = cli.client.NewAgentSession(*sessionID)
|
||||
fmt.Printf("%s✓ Using existing session: %s%s\n\n", ColorGreen, *sessionID, ColorReset)
|
||||
}
|
||||
|
||||
cli.Run()
|
||||
}
|
||||
|
||||
// NewCLI creates a new CLI instance
|
||||
func NewCLI(config *Config) *CLI {
|
||||
clientOptions := []client.ClientOption{
|
||||
client.WithTimeout(120 * time.Second),
|
||||
}
|
||||
if config.Token != "" {
|
||||
clientOptions = append(clientOptions, client.WithToken(config.Token))
|
||||
}
|
||||
|
||||
return &CLI{
|
||||
client: client.NewClient(config.BaseURL, clientOptions...),
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts the interactive CLI
|
||||
func (cli *CLI) Run() {
|
||||
cli.printWelcome()
|
||||
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
|
||||
for {
|
||||
fmt.Print(cli.getPrompt())
|
||||
|
||||
input, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
fmt.Printf("%sError reading input: %v%s\n", ColorRed, err, ColorReset)
|
||||
continue
|
||||
}
|
||||
|
||||
input = strings.TrimSpace(input)
|
||||
if input == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if !cli.handleCommand(input) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\n%sGoodbye!%s\n", ColorCyan, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) getPrompt() string {
|
||||
if cli.currentSessionID != "" {
|
||||
return fmt.Sprintf("%s[Session: %s]%s > ", ColorBlue, cli.currentSessionID[:8], ColorReset)
|
||||
}
|
||||
return fmt.Sprintf("%s[No Session]%s > ", ColorYellow, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) printWelcome() {
|
||||
fmt.Printf("\n%s╔════════════════════════════════════════════════════════════╗%s\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("%s║ WeKnora Agent QA Testing Tool ║%s\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("%s╚════════════════════════════════════════════════════════════╝%s\n\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("%sAvailable commands:%s\n", ColorBold, ColorReset)
|
||||
fmt.Printf(" %snew%s - Create a new session\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %sask <query>%s - Ask agent a question\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %sinfo%s - Show current session info\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %ssessions%s - List all sessions\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %sswitch <id>%s - Switch to another session\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %shelp%s - Show this help message\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %sexit/quit%s - Exit the program\n\n", ColorGreen, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) handleCommand(input string) bool {
|
||||
parts := strings.SplitN(input, " ", 2)
|
||||
command := strings.ToLower(parts[0])
|
||||
|
||||
switch command {
|
||||
case "help":
|
||||
cli.printWelcome()
|
||||
case "new":
|
||||
cli.createNewSession()
|
||||
case "ask":
|
||||
if len(parts) < 2 {
|
||||
fmt.Printf("%sUsage: ask <your question>%s\n", ColorRed, ColorReset)
|
||||
return true
|
||||
}
|
||||
cli.askQuestion(parts[1])
|
||||
case "info":
|
||||
cli.showSessionInfo()
|
||||
case "sessions":
|
||||
cli.listSessions()
|
||||
case "switch":
|
||||
if len(parts) < 2 {
|
||||
fmt.Printf("%sUsage: switch <session_id>%s\n", ColorRed, ColorReset)
|
||||
return true
|
||||
}
|
||||
cli.switchSession(parts[1])
|
||||
case "exit", "quit":
|
||||
return false
|
||||
default:
|
||||
fmt.Printf("%sUnknown command: %s. Type 'help' for available commands.%s\n", ColorRed, command, ColorReset)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (cli *CLI) createNewSession() {
|
||||
if cli.config.KnowledgeBaseID == "" {
|
||||
fmt.Printf("%sError: Knowledge base ID is required. Use -kb flag.%s\n", ColorRed, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("%sCreating new session...%s\n", ColorYellow, ColorReset)
|
||||
|
||||
ctx := context.Background()
|
||||
request := &client.CreateSessionRequest{
|
||||
KnowledgeBaseID: cli.config.KnowledgeBaseID,
|
||||
SessionStrategy: &client.SessionStrategy{
|
||||
MaxRounds: 10,
|
||||
EnableRewrite: false,
|
||||
FallbackStrategy: "default",
|
||||
EmbeddingTopK: 10,
|
||||
},
|
||||
}
|
||||
|
||||
session, err := cli.client.CreateSession(ctx, request)
|
||||
if err != nil {
|
||||
fmt.Printf("%s✗ Failed to create session: %v%s\n", ColorRed, err, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
cli.currentSessionID = session.ID
|
||||
cli.agentSession = cli.client.NewAgentSession(session.ID)
|
||||
|
||||
fmt.Printf("%s✓ Session created successfully!%s\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" Session ID: %s%s%s\n", ColorCyan, session.ID, ColorReset)
|
||||
fmt.Printf(" Knowledge Base: %s%s%s\n", ColorCyan, session.KnowledgeBaseID, ColorReset)
|
||||
fmt.Printf(" Created At: %s%s%s\n\n", ColorCyan, session.CreatedAt, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) askQuestion(query string) {
|
||||
if cli.agentSession == nil {
|
||||
fmt.Printf("%sError: No active session. Create one with 'new' command.%s\n", ColorRed, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("\n%s╔═══ Agent Processing ═══╗%s\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("%sQuery: %s%s%s\n\n", ColorBold, ColorWhite, query, ColorReset)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
var (
|
||||
thinkingContent string
|
||||
finalAnswer string
|
||||
references []*client.SearchResult
|
||||
toolCalls []string
|
||||
reflections []string
|
||||
)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
err := cli.agentSession.Ask(ctx, query, func(resp *client.AgentStreamResponse) error {
|
||||
switch resp.ResponseType {
|
||||
case client.AgentResponseTypeThinking:
|
||||
thinkingContent += resp.Content
|
||||
if resp.Done {
|
||||
fmt.Printf("%s💭 Thinking:%s %s\n\n", ColorYellow, ColorReset, thinkingContent)
|
||||
thinkingContent = ""
|
||||
}
|
||||
|
||||
case client.AgentResponseTypeToolCall:
|
||||
toolName := "Unknown"
|
||||
if resp.Data != nil {
|
||||
if name, ok := resp.Data["tool_name"].(string); ok {
|
||||
toolName = name
|
||||
}
|
||||
}
|
||||
toolCalls = append(toolCalls, toolName)
|
||||
fmt.Printf("%s🔧 Tool Call:%s %s\n", ColorMagenta, ColorReset, toolName)
|
||||
if resp.Data != nil {
|
||||
if args, ok := resp.Data["arguments"]; ok {
|
||||
fmt.Printf(" Arguments: %v\n", args)
|
||||
}
|
||||
}
|
||||
|
||||
case client.AgentResponseTypeToolResult:
|
||||
fmt.Printf("%s✓ Tool Result:%s\n", ColorGreen, ColorReset)
|
||||
fmt.Printf(" %s\n\n", resp.Content)
|
||||
|
||||
case client.AgentResponseTypeReferences:
|
||||
if resp.KnowledgeReferences != nil {
|
||||
references = append(references, resp.KnowledgeReferences...)
|
||||
fmt.Printf(
|
||||
"%s📚 Knowledge References:%s Found %d reference(s)\n",
|
||||
ColorCyan,
|
||||
ColorReset,
|
||||
len(resp.KnowledgeReferences),
|
||||
)
|
||||
for i, ref := range resp.KnowledgeReferences {
|
||||
fmt.Printf(" %d. [Score: %.3f] %s\n", i+1, ref.Score, truncateString(ref.Content, 80))
|
||||
fmt.Printf(" Knowledge: %s (Chunk: %d)\n", ref.KnowledgeTitle, ref.ChunkIndex)
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
case client.AgentResponseTypeAnswer:
|
||||
finalAnswer += resp.Content
|
||||
if resp.Done {
|
||||
fmt.Printf("%s📝 Final Answer:%s\n", ColorGreen, ColorReset)
|
||||
fmt.Printf("%s\n\n", finalAnswer)
|
||||
}
|
||||
|
||||
case client.AgentResponseTypeReflection:
|
||||
if resp.Done && resp.Content != "" {
|
||||
reflections = append(reflections, resp.Content)
|
||||
fmt.Printf("%s🤔 Reflection:%s %s\n\n", ColorBlue, ColorReset, resp.Content)
|
||||
}
|
||||
|
||||
case client.AgentResponseTypeError:
|
||||
fmt.Printf("%s✗ Error:%s %s\n\n", ColorRed, ColorReset, resp.Content)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("%s✗ Agent QA failed: %v%s\n", ColorRed, err, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
duration := time.Since(startTime)
|
||||
|
||||
// Print summary
|
||||
fmt.Printf("%s╔═══ Summary ═══╗%s\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("Duration: %s%.2fs%s\n", ColorCyan, duration.Seconds(), ColorReset)
|
||||
fmt.Printf("Tool Calls: %s%d%s", ColorCyan, len(toolCalls), ColorReset)
|
||||
if len(toolCalls) > 0 {
|
||||
fmt.Printf(" (%s)", strings.Join(toolCalls, ", "))
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Printf("References: %s%d%s\n", ColorCyan, len(references), ColorReset)
|
||||
fmt.Printf("Reflections: %s%d%s\n", ColorCyan, len(reflections), ColorReset)
|
||||
fmt.Printf("%s╚═══════════════╝%s\n\n", ColorCyan, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) showSessionInfo() {
|
||||
if cli.currentSessionID == "" {
|
||||
fmt.Printf("%sNo active session.%s\n", ColorYellow, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
session, err := cli.client.GetSession(ctx, cli.currentSessionID)
|
||||
if err != nil {
|
||||
fmt.Printf("%s✗ Failed to get session info: %v%s\n", ColorRed, err, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("\n%s╔═══ Session Information ═══╗%s\n", ColorCyan, ColorReset)
|
||||
fmt.Printf("Session ID: %s%s%s\n", ColorWhite, session.ID, ColorReset)
|
||||
fmt.Printf("Title: %s%s%s\n", ColorWhite, session.Title, ColorReset)
|
||||
fmt.Printf("Knowledge Base ID: %s%s%s\n", ColorWhite, session.KnowledgeBaseID, ColorReset)
|
||||
fmt.Printf("Max Rounds: %s%d%s\n", ColorWhite, session.MaxRounds, ColorReset)
|
||||
fmt.Printf("Enable Rewrite: %s%v%s\n", ColorWhite, session.EnableRewrite, ColorReset)
|
||||
fmt.Printf("Summary Model: %s%s%s\n", ColorWhite, session.SummaryModelID, ColorReset)
|
||||
fmt.Printf("Created At: %s%s%s\n", ColorWhite, session.CreatedAt, ColorReset)
|
||||
fmt.Printf("Updated At: %s%s%s\n", ColorWhite, session.UpdatedAt, ColorReset)
|
||||
fmt.Printf("%s╚═══════════════════════════╝%s\n\n", ColorCyan, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) listSessions() {
|
||||
ctx := context.Background()
|
||||
sessions, total, err := cli.client.GetSessionsByTenant(ctx, 1, 10)
|
||||
if err != nil {
|
||||
fmt.Printf("%s✗ Failed to list sessions: %v%s\n", ColorRed, err, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("\n%s╔═══ Sessions (Total: %d) ═══╗%s\n", ColorCyan, total, ColorReset)
|
||||
for i, session := range sessions {
|
||||
marker := " "
|
||||
if session.ID == cli.currentSessionID {
|
||||
marker = "►"
|
||||
}
|
||||
fmt.Printf("%s%s %d. %s%s\n", ColorGreen, marker, i+1, session.ID, ColorReset)
|
||||
fmt.Printf(" Title: %s\n", session.Title)
|
||||
fmt.Printf(" KB: %s\n", session.KnowledgeBaseID)
|
||||
fmt.Printf(" Created: %s\n", session.CreatedAt)
|
||||
fmt.Println()
|
||||
}
|
||||
fmt.Printf("%s╚════════════════════════════╝%s\n\n", ColorCyan, ColorReset)
|
||||
}
|
||||
|
||||
func (cli *CLI) switchSession(sessionID string) {
|
||||
ctx := context.Background()
|
||||
session, err := cli.client.GetSession(ctx, sessionID)
|
||||
if err != nil {
|
||||
fmt.Printf("%s✗ Failed to switch session: %v%s\n", ColorRed, err, ColorReset)
|
||||
return
|
||||
}
|
||||
|
||||
cli.currentSessionID = session.ID
|
||||
cli.agentSession = cli.client.NewAgentSession(session.ID)
|
||||
|
||||
fmt.Printf("%s✓ Switched to session: %s%s\n", ColorGreen, session.ID, ColorReset)
|
||||
}
|
||||
|
||||
func truncateString(s string, maxLen int) string {
|
||||
s = strings.ReplaceAll(s, "\n", " ")
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# WeKnora Agent Test Tool - Quick Start Script
|
||||
# This script helps you quickly build and run the agent test tool
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Default values
|
||||
DEFAULT_URL="http://localhost:8080"
|
||||
BINARY_NAME="agent_test"
|
||||
|
||||
# Function to print colored output
|
||||
print_info() {
|
||||
echo -e "${BLUE}ℹ${NC} $1"
|
||||
}
|
||||
|
||||
print_success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
print_warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
}
|
||||
|
||||
# Function to build the binary
|
||||
build() {
|
||||
print_info "Building agent test tool..."
|
||||
if go build -o "$BINARY_NAME" .; then
|
||||
print_success "Build successful!"
|
||||
return 0
|
||||
else
|
||||
print_error "Build failed!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to show usage
|
||||
usage() {
|
||||
cat << EOF
|
||||
WeKnora Agent Test Tool - Quick Start Script
|
||||
|
||||
Usage:
|
||||
$0 [OPTIONS]
|
||||
|
||||
Options:
|
||||
-u, --url URL WeKnora API URL (default: $DEFAULT_URL)
|
||||
-k, --kb KB_ID Knowledge Base ID
|
||||
-s, --session ID Existing Session ID
|
||||
-t, --token TOKEN API authentication token
|
||||
-b, --build-only Only build, don't run
|
||||
-h, --help Show this help message
|
||||
|
||||
Examples:
|
||||
# Build and run with a knowledge base
|
||||
$0 -k kb_123456
|
||||
|
||||
# Use a different server URL
|
||||
$0 -u https://api.weknora.com -k kb_123456 -t mytoken
|
||||
|
||||
# Continue an existing session
|
||||
$0 -s sess_123456
|
||||
|
||||
# Only build the binary
|
||||
$0 --build-only
|
||||
|
||||
Environment Variables:
|
||||
WEKNORA_URL Default API URL
|
||||
WEKNORA_KB_ID Default Knowledge Base ID
|
||||
WEKNORA_TOKEN Default API Token
|
||||
WEKNORA_SESSION_ID Default Session ID
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# Parse command line arguments
|
||||
URL="${WEKNORA_URL:-$DEFAULT_URL}"
|
||||
KB_ID="${WEKNORA_KB_ID:-}"
|
||||
SESSION_ID="${WEKNORA_SESSION_ID:-}"
|
||||
TOKEN="${WEKNORA_TOKEN:-}"
|
||||
BUILD_ONLY=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-u|--url)
|
||||
URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
-k|--kb)
|
||||
KB_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s|--session)
|
||||
SESSION_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
-t|--token)
|
||||
TOKEN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-b|--build-only)
|
||||
BUILD_ONLY=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown option: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Build the binary
|
||||
if ! build; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Exit if build-only mode
|
||||
if [ "$BUILD_ONLY" = true ]; then
|
||||
print_info "Build-only mode. Binary created: ./$BINARY_NAME"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Validate parameters
|
||||
if [ -z "$SESSION_ID" ] && [ -z "$KB_ID" ]; then
|
||||
print_warning "Neither session ID nor knowledge base ID provided."
|
||||
print_info "You can create a new session after starting the tool."
|
||||
fi
|
||||
|
||||
# Build command arguments
|
||||
ARGS="-url $URL"
|
||||
|
||||
if [ -n "$KB_ID" ]; then
|
||||
ARGS="$ARGS -kb $KB_ID"
|
||||
fi
|
||||
|
||||
if [ -n "$SESSION_ID" ]; then
|
||||
ARGS="$ARGS -session $SESSION_ID"
|
||||
fi
|
||||
|
||||
if [ -n "$TOKEN" ]; then
|
||||
ARGS="$ARGS -token $TOKEN"
|
||||
fi
|
||||
|
||||
# Show configuration
|
||||
echo ""
|
||||
print_info "Configuration:"
|
||||
echo " URL: $URL"
|
||||
[ -n "$KB_ID" ] && echo " Knowledge Base: $KB_ID"
|
||||
[ -n "$SESSION_ID" ] && echo " Session: $SESSION_ID"
|
||||
[ -n "$TOKEN" ] && echo " Token: [provided]"
|
||||
echo ""
|
||||
|
||||
# Run the tool
|
||||
print_info "Starting agent test tool..."
|
||||
echo ""
|
||||
./$BINARY_NAME $ARGS
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test Scenarios for WeKnora Agent QA
|
||||
# This script contains various test scenarios you can run
|
||||
|
||||
set -e
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
print_scenario() {
|
||||
echo -e "\n${BLUE}═══════════════════════════════════════════════${NC}"
|
||||
echo -e "${BLUE} $1${NC}"
|
||||
echo -e "${BLUE}═══════════════════════════════════════════════${NC}\n"
|
||||
}
|
||||
|
||||
# Check if binary exists
|
||||
if [ ! -f "./agent_test" ]; then
|
||||
echo "Building agent_test..."
|
||||
make build
|
||||
fi
|
||||
|
||||
# Load environment variables if .env exists
|
||||
if [ -f ".env" ]; then
|
||||
export $(cat .env | grep -v '^#' | xargs)
|
||||
fi
|
||||
|
||||
# Check required variables
|
||||
if [ -z "$WEKNORA_KB_ID" ]; then
|
||||
echo "Error: WEKNORA_KB_ID is not set"
|
||||
echo "Please set it in .env file or export WEKNORA_KB_ID=your_kb_id"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
URL="${WEKNORA_URL:-http://localhost:8080}"
|
||||
KB_ID="$WEKNORA_KB_ID"
|
||||
TOKEN="${WEKNORA_TOKEN:-}"
|
||||
|
||||
# Scenario 1: Basic Q&A
|
||||
scenario_basic() {
|
||||
print_scenario "Scenario 1: Basic Question & Answer"
|
||||
echo -e "${YELLOW}This will test basic agent Q&A functionality${NC}\n"
|
||||
echo "Try asking: What is machine learning?"
|
||||
echo ""
|
||||
|
||||
if [ -n "$TOKEN" ]; then
|
||||
./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN"
|
||||
else
|
||||
./agent_test -url "$URL" -kb "$KB_ID"
|
||||
fi
|
||||
}
|
||||
|
||||
# Scenario 2: Complex query with tool usage
|
||||
scenario_complex() {
|
||||
print_scenario "Scenario 2: Complex Query with Tool Usage"
|
||||
echo -e "${YELLOW}This will test agent's ability to use tools${NC}\n"
|
||||
echo "Try asking: Search for information about neural networks and explain the key concepts"
|
||||
echo ""
|
||||
|
||||
if [ -n "$TOKEN" ]; then
|
||||
./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN"
|
||||
else
|
||||
./agent_test -url "$URL" -kb "$KB_ID"
|
||||
fi
|
||||
}
|
||||
|
||||
# Scenario 3: Multi-turn conversation
|
||||
scenario_multiturn() {
|
||||
print_scenario "Scenario 3: Multi-turn Conversation"
|
||||
echo -e "${YELLOW}This will test agent's context awareness${NC}\n"
|
||||
echo "Try this sequence:"
|
||||
echo " 1. What is supervised learning?"
|
||||
echo " 2. Can you give me some examples?"
|
||||
echo " 3. How does it compare to unsupervised learning?"
|
||||
echo ""
|
||||
|
||||
if [ -n "$TOKEN" ]; then
|
||||
./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN"
|
||||
else
|
||||
./agent_test -url "$URL" -kb "$KB_ID"
|
||||
fi
|
||||
}
|
||||
|
||||
# Show menu
|
||||
show_menu() {
|
||||
echo -e "\n${GREEN}WeKnora Agent Test Scenarios${NC}\n"
|
||||
echo "Select a test scenario to run:"
|
||||
echo " 1) Basic Q&A"
|
||||
echo " 2) Complex Query with Tool Usage"
|
||||
echo " 3) Multi-turn Conversation"
|
||||
echo " 4) Custom (manual testing)"
|
||||
echo " 5) Exit"
|
||||
echo ""
|
||||
read -p "Enter your choice [1-5]: " choice
|
||||
|
||||
case $choice in
|
||||
1) scenario_basic ;;
|
||||
2) scenario_complex ;;
|
||||
3) scenario_multiturn ;;
|
||||
4)
|
||||
print_scenario "Custom Testing"
|
||||
if [ -n "$TOKEN" ]; then
|
||||
./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN"
|
||||
else
|
||||
./agent_test -url "$URL" -kb "$KB_ID"
|
||||
fi
|
||||
;;
|
||||
5)
|
||||
echo "Goodbye!"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Invalid choice. Please try again."
|
||||
show_menu
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Main
|
||||
if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then
|
||||
echo "Usage: $0 [scenario_number]"
|
||||
echo ""
|
||||
echo "Available scenarios:"
|
||||
echo " 1 - Basic Q&A"
|
||||
echo " 2 - Complex Query with Tool Usage"
|
||||
echo " 3 - Multi-turn Conversation"
|
||||
echo " 4 - Custom (manual testing)"
|
||||
echo ""
|
||||
echo "If no argument is provided, an interactive menu will be shown."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If scenario number is provided as argument
|
||||
if [ -n "$1" ]; then
|
||||
case $1 in
|
||||
1) scenario_basic ;;
|
||||
2) scenario_complex ;;
|
||||
3) scenario_multiturn ;;
|
||||
4)
|
||||
if [ -n "$TOKEN" ]; then
|
||||
./agent_test -url "$URL" -kb "$KB_ID" -token "$TOKEN"
|
||||
else
|
||||
./agent_test -url "$URL" -kb "$KB_ID"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Invalid scenario number: $1"
|
||||
echo "Use --help to see available scenarios"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
else
|
||||
# Show interactive menu
|
||||
show_menu
|
||||
fi
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package agent
|
||||
|
||||
const (
|
||||
DefaultAgentTemperature = 0.7
|
||||
DefaultAgentMaxIterations = 20
|
||||
// DefaultAgentTemperature is the default temperature for the agent
|
||||
DefaultAgentTemperature = 0.7
|
||||
// DefaultAgentMaxIterations is the default maximum number of iterations for the agent
|
||||
DefaultAgentMaxIterations = 20
|
||||
// DefaultAgentReflectionEnabled is the default whether to enable reflection for the agent
|
||||
DefaultAgentReflectionEnabled = false
|
||||
DefaultUseCustomSystemPrompt = false
|
||||
// DefaultUseCustomSystemPrompt is the default whether to use custom system prompt for the agent
|
||||
DefaultUseCustomSystemPrompt = false
|
||||
)
|
||||
|
||||
@@ -96,7 +96,7 @@ func (e *AgentEngine) Execute(
|
||||
}
|
||||
|
||||
// Build system prompt using progressive RAG prompt
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(
|
||||
systemPrompt := BuildSystemPrompt(
|
||||
e.knowledgeBasesInfo,
|
||||
e.config.WebSearchEnabled,
|
||||
e.systemPromptTemplate,
|
||||
@@ -771,7 +771,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
})
|
||||
|
||||
// Build messages with all context
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(
|
||||
systemPrompt := BuildSystemPrompt(
|
||||
e.knowledgeBasesInfo,
|
||||
e.config.WebSearchEnabled,
|
||||
e.systemPromptTemplate,
|
||||
|
||||
@@ -209,8 +209,8 @@ func renderPromptPlaceholdersWithStatus(
|
||||
return result
|
||||
}
|
||||
|
||||
// BuildProgressiveRAGSystemPromptWithWeb builds the progressive RAG system prompt with web search enabled
|
||||
func BuildProgressiveRAGSystemPromptWithWeb(
|
||||
// BuildSystemPromptWithWeb builds the progressive RAG system prompt with web search enabled
|
||||
func BuildSystemPromptWithWeb(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
@@ -224,8 +224,8 @@ func BuildProgressiveRAGSystemPromptWithWeb(
|
||||
return renderPromptPlaceholdersWithStatus(template, knowledgeBases, true, currentTime)
|
||||
}
|
||||
|
||||
// BuildProgressiveRAGSystemPromptWithoutWeb builds the progressive RAG system prompt without web search
|
||||
func BuildProgressiveRAGSystemPromptWithoutWeb(
|
||||
// BuildSystemPromptWithoutWeb builds the progressive RAG system prompt without web search
|
||||
func BuildSystemPromptWithoutWeb(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
@@ -241,15 +241,15 @@ func BuildProgressiveRAGSystemPromptWithoutWeb(
|
||||
|
||||
// BuildProgressiveRAGSystemPrompt builds the progressive RAG system prompt based on web search status
|
||||
// This is the main function to use - it automatically selects the appropriate version
|
||||
func BuildProgressiveRAGSystemPrompt(
|
||||
func BuildSystemPrompt(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
webSearchEnabled bool,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
if webSearchEnabled {
|
||||
return BuildProgressiveRAGSystemPromptWithWeb(knowledgeBases, systemPromptTemplate...)
|
||||
return BuildSystemPromptWithWeb(knowledgeBases, systemPromptTemplate...)
|
||||
}
|
||||
return BuildProgressiveRAGSystemPromptWithoutWeb(knowledgeBases, systemPromptTemplate...)
|
||||
return BuildSystemPromptWithoutWeb(knowledgeBases, systemPromptTemplate...)
|
||||
}
|
||||
|
||||
// ProgressiveRAGSystemPromptWithWeb is the progressive RAG system prompt template with web search enabled
|
||||
|
||||
@@ -98,6 +98,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
}
|
||||
}
|
||||
|
||||
// Check if knowledge IDs are valid
|
||||
if len(knowledgeIDs) == 0 {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
@@ -116,6 +117,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
var mu sync.Mutex
|
||||
results := make(map[string]*docInfo)
|
||||
|
||||
// Concurrently get info for each knowledge ID
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
wg.Add(1)
|
||||
go func(id string) {
|
||||
@@ -195,10 +197,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
for i, doc := range successDocs {
|
||||
k := doc.knowledge
|
||||
|
||||
// Get file type icon
|
||||
typeIcon := getTypeIcon(k.Type, k.FileType)
|
||||
|
||||
output += fmt.Sprintf("【文档 #%d】 %s\n", i+1, typeIcon)
|
||||
output += fmt.Sprintf("【文档 #%d】\n", i+1)
|
||||
output += fmt.Sprintf(" ID: %s\n", k.ID)
|
||||
output += fmt.Sprintf(" 标题: %s\n", k.Title)
|
||||
|
||||
@@ -240,7 +239,6 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
"parse_status": k.ParseStatus,
|
||||
"chunk_count": doc.chunkCount,
|
||||
"metadata": k.GetMetadata(),
|
||||
"type_icon": typeIcon,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -264,36 +262,6 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func getTypeIcon(knowledgeType, fileType string) string {
|
||||
switch knowledgeType {
|
||||
case "file":
|
||||
switch fileType {
|
||||
case "pdf", "PDF":
|
||||
return "📄 PDF"
|
||||
case "docx", "doc", "DOCX", "DOC":
|
||||
return "📝 Word"
|
||||
case "txt", "TXT":
|
||||
return "📃 Text"
|
||||
case "md", "MD", "markdown":
|
||||
return "📋 Markdown"
|
||||
case "xlsx", "xls", "XLSX", "XLS":
|
||||
return "📊 Excel"
|
||||
case "pptx", "ppt", "PPTX", "PPT":
|
||||
return "📊 PowerPoint"
|
||||
default:
|
||||
return "📄 File"
|
||||
}
|
||||
case "url":
|
||||
return "🌐 URL"
|
||||
case "passage":
|
||||
return "📝 Passage"
|
||||
default:
|
||||
return "📄 Document"
|
||||
}
|
||||
}
|
||||
|
||||
func formatSource(knowledgeType, source string) string {
|
||||
switch knowledgeType {
|
||||
case "file":
|
||||
|
||||
@@ -1173,44 +1173,6 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
output += fmt.Sprintf(" 已召回: %d 个 (%.1f%%)\n", retrievedCount, percentage)
|
||||
output += fmt.Sprintf(" 未召回: %d 个\n", remaining)
|
||||
|
||||
if remaining > 0 {
|
||||
output += " 建议: 使用 list_knowledge_chunks 工具获取完整内容\n"
|
||||
|
||||
// Find missing chunk ranges (gaps in retrieved chunks)
|
||||
missingRanges := t.findMissingChunkRanges(retrievedChunks, int(totalChunks))
|
||||
|
||||
if len(missingRanges) == 0 {
|
||||
// No gaps found (shouldn't happen if remaining > 0, but handle it)
|
||||
output += fmt.Sprintf(
|
||||
" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n",
|
||||
knowledgeID,
|
||||
totalChunks,
|
||||
)
|
||||
} else if len(missingRanges) == 1 && missingRanges[0].start == 0 && missingRanges[0].end == int(totalChunks)-1 {
|
||||
// All chunks are missing (shouldn't happen, but handle it)
|
||||
output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks)
|
||||
} else {
|
||||
// Suggest getting each missing range
|
||||
for idx, r := range missingRanges {
|
||||
rangeSize := r.end - r.start + 1
|
||||
if rangeSize <= 100 {
|
||||
// Small range, get all at once
|
||||
output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个) → list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=%d)\n",
|
||||
idx+1, r.start, r.end, rangeSize, knowledgeID, r.start, rangeSize)
|
||||
} else {
|
||||
// Large range, suggest getting in batches
|
||||
output += fmt.Sprintf(" - 区间 %d: chunk_index %d-%d (%d 个,建议分批获取):\n",
|
||||
idx+1, r.start, r.end, rangeSize)
|
||||
output += fmt.Sprintf(" 首次: list_knowledge_chunks(knowledge_id=\"%s\", offset=%d, limit=100)\n",
|
||||
knowledgeID, r.start)
|
||||
if rangeSize > 100 {
|
||||
output += " 继续: 根据返回结果调整 offset 继续获取剩余内容\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output += "\n"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -168,6 +168,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i
|
||||
}, nil
|
||||
}
|
||||
|
||||
// lookupKnowledgeTitle looks up the title of a knowledge document
|
||||
func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, knowledgeID string) string {
|
||||
if t.knowledgeService == nil {
|
||||
return ""
|
||||
@@ -179,6 +180,7 @@ func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, know
|
||||
return strings.TrimSpace(knowledge.Title)
|
||||
}
|
||||
|
||||
// buildOutput builds the output for the list knowledge chunks tool
|
||||
func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
knowledgeID string,
|
||||
knowledgeTitle string,
|
||||
@@ -226,6 +228,7 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// summarizeContent summarizes the content of a chunk
|
||||
func summarizeContent(content string) string {
|
||||
cleaned := strings.TrimSpace(content)
|
||||
if cleaned == "" {
|
||||
|
||||
@@ -85,7 +85,7 @@ func (t *QueryKnowledgeGraphTool) Parameters() map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
// Execute queries the knowledge graph with concurrent KB processing
|
||||
// Execute performs the knowledge graph query with concurrent KB processing
|
||||
func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
|
||||
// Extract knowledge_base_ids array
|
||||
kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{})
|
||||
@@ -394,14 +394,10 @@ func buildGraphVisualizationData(
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Extract actual entities and relations when graph extraction is fully implemented
|
||||
// For now, create placeholder structure
|
||||
|
||||
return map[string]interface{}{
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"total_nodes": len(nodes),
|
||||
"total_edges": len(edges),
|
||||
"visualization_note": "完整的图可视化功能开发中,当前显示文档节点",
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"total_nodes": len(nodes),
|
||||
"total_edges": len(edges),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// SequentialThinkingTool implements a dynamic and reflective problem-solving tool
|
||||
// SequentialThinkingTool is a dynamic and reflective problem-solving tool
|
||||
// This tool helps analyze problems through a flexible thinking process that can adapt and evolve
|
||||
type SequentialThinkingTool struct {
|
||||
BaseTool
|
||||
|
||||
@@ -19,29 +19,31 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
webFetchTimeout = 60 * time.Second
|
||||
webFetchMaxChars = 100000
|
||||
webFetchTimeout = 60 * time.Second // timeout for web fetch
|
||||
webFetchMaxChars = 100000 // maximum number of characters to fetch
|
||||
)
|
||||
|
||||
// webFetchParams is the parameters for the web fetch tool
|
||||
type webFetchParams struct {
|
||||
URL string
|
||||
Prompt string
|
||||
}
|
||||
|
||||
// webFetchItemResult is the result for a web fetch item
|
||||
type webFetchItemResult struct {
|
||||
output string
|
||||
data map[string]interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
// WebFetchTool 拉取网页内容并根据提示执行总结
|
||||
// WebFetchTool fetches web page content and summarizes it using an LLM
|
||||
type WebFetchTool struct {
|
||||
BaseTool
|
||||
client *http.Client
|
||||
chatModel chat.Chat
|
||||
}
|
||||
|
||||
// NewWebFetchTool 创建 web_fetch 工具实例
|
||||
// NewWebFetchTool creates a new web_fetch tool instance
|
||||
func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool {
|
||||
description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM.
|
||||
|
||||
@@ -226,6 +228,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{})
|
||||
}, nil
|
||||
}
|
||||
|
||||
// parseParams parses the parameters for a web fetch item
|
||||
func (t *WebFetchTool) parseParams(item interface{}) webFetchParams {
|
||||
params := webFetchParams{}
|
||||
if m, ok := item.(map[string]interface{}); ok {
|
||||
@@ -239,6 +242,7 @@ func (t *WebFetchTool) parseParams(item interface{}) webFetchParams {
|
||||
return params
|
||||
}
|
||||
|
||||
// validateParams validates the parameters for a web fetch item
|
||||
func (t *WebFetchTool) validateParams(p webFetchParams) error {
|
||||
if p.URL == "" {
|
||||
return fmt.Errorf("url is required")
|
||||
@@ -252,6 +256,7 @@ func (t *WebFetchTool) validateParams(p webFetchParams) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// executeFetch executes a web fetch item
|
||||
func (t *WebFetchTool) executeFetch(
|
||||
ctx context.Context,
|
||||
params webFetchParams,
|
||||
@@ -294,6 +299,7 @@ func (t *WebFetchTool) executeFetch(
|
||||
return output, resultData, summaryErr
|
||||
}
|
||||
|
||||
// normalizeGitHubURL normalizes a GitHub URL
|
||||
func (t *WebFetchTool) normalizeGitHubURL(source string) string {
|
||||
if strings.Contains(source, "github.com") && strings.Contains(source, "/blob/") {
|
||||
source = strings.Replace(source, "github.com", "raw.githubusercontent.com", 1)
|
||||
@@ -302,6 +308,7 @@ func (t *WebFetchTool) normalizeGitHubURL(source string) string {
|
||||
return source
|
||||
}
|
||||
|
||||
// processWithLLM processes the content with an LLM
|
||||
func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams, content string) (string, error) {
|
||||
if t.chatModel == nil {
|
||||
return "", fmt.Errorf("chat model not available for web_fetch")
|
||||
@@ -336,6 +343,7 @@ func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams
|
||||
return strings.TrimSpace(response.Content), nil
|
||||
}
|
||||
|
||||
// buildOutputText builds the output text for a web fetch item
|
||||
func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, summary string, summaryErr error) string {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(fmt.Sprintf("URL: %s\n", params.URL))
|
||||
@@ -354,6 +362,7 @@ func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, su
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// fetchHTMLContent fetches the HTML content for a web fetch item
|
||||
func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) {
|
||||
html, err := t.fetchWithChromedp(ctx, targetURL)
|
||||
if err == nil && strings.TrimSpace(html) != "" {
|
||||
@@ -375,6 +384,7 @@ func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (
|
||||
return html, "http", nil
|
||||
}
|
||||
|
||||
// fetchWithChromedp fetches the HTML content with Chromedp
|
||||
func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) {
|
||||
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL)
|
||||
|
||||
@@ -414,6 +424,7 @@ func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string)
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// fetchWithHTTP fetches the HTML content with HTTP
|
||||
func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) {
|
||||
resp, err := t.fetchWithTimeout(ctx, targetURL)
|
||||
if err != nil {
|
||||
@@ -434,6 +445,7 @@ func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (str
|
||||
return string(htmlBytes), nil
|
||||
}
|
||||
|
||||
// fetchWithTimeout fetches the HTML content with a timeout
|
||||
func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||
if err != nil {
|
||||
@@ -451,6 +463,7 @@ func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (
|
||||
return t.client.Do(req)
|
||||
}
|
||||
|
||||
// convertHTMLToText converts the HTML content to text
|
||||
func (t *WebFetchTool) convertHTMLToText(html string) string {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
@@ -469,6 +482,7 @@ func (t *WebFetchTool) convertHTMLToText(html string) string {
|
||||
return strings.TrimSpace(result)
|
||||
}
|
||||
|
||||
// processNode processes a node in the HTML content
|
||||
func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Builder) {
|
||||
s.Contents().Each(func(i int, node *goquery.Selection) {
|
||||
nodeName := goquery.NodeName(node)
|
||||
@@ -578,6 +592,7 @@ func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Build
|
||||
})
|
||||
}
|
||||
|
||||
// basicTextExtraction extracts the text from the HTML content
|
||||
func (t *WebFetchTool) basicTextExtraction(html string) string {
|
||||
re := regexp.MustCompile(`<[^>]*>`)
|
||||
text := re.ReplaceAllString(html, " ")
|
||||
|
||||
@@ -11,19 +11,23 @@ import (
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
)
|
||||
|
||||
// Neo4jRepository is a repository for Neo4j
|
||||
type Neo4jRepository struct {
|
||||
driver neo4j.Driver
|
||||
nodePrefix string
|
||||
}
|
||||
|
||||
// NewNeo4jRepository creates a new Neo4j repository
|
||||
func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository {
|
||||
return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"}
|
||||
}
|
||||
|
||||
// _remove_hyphen removes hyphens from a string
|
||||
func _remove_hyphen(s string) string {
|
||||
return strings.ReplaceAll(s, "-", "_")
|
||||
}
|
||||
|
||||
// Labels returns the labels for a namespace
|
||||
func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
|
||||
res := make([]string, 0)
|
||||
for _, label := range namespace.Labels() {
|
||||
@@ -32,12 +36,13 @@ func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string {
|
||||
return res
|
||||
}
|
||||
|
||||
// Label returns the label for a namespace
|
||||
func (n *Neo4jRepository) Label(namespace types.NameSpace) string {
|
||||
labels := n.Labels(namespace)
|
||||
return strings.Join(labels, ":")
|
||||
}
|
||||
|
||||
// AddGraph implements interfaces.RetrieveGraphRepository.
|
||||
// AddGraph adds a graph to the Neo4j repository
|
||||
func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
@@ -51,11 +56,13 @@ func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil
|
||||
}
|
||||
|
||||
// addGraph adds a graph to the Neo4j repository
|
||||
func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error {
|
||||
session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
|
||||
defer session.Close(ctx)
|
||||
|
||||
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) {
|
||||
// Node import query
|
||||
node_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node
|
||||
@@ -76,6 +83,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil, fmt.Errorf("failed to create nodes: %v", err)
|
||||
}
|
||||
|
||||
// Relationship import query
|
||||
rel_import_query := `
|
||||
UNWIND $data AS row
|
||||
CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source
|
||||
@@ -106,7 +114,7 @@ func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpac
|
||||
return nil
|
||||
}
|
||||
|
||||
// DelGraph implements interfaces.RetrieveGraphRepository.
|
||||
// DelGraph deletes a graph from the Neo4j repository
|
||||
func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
@@ -152,6 +160,7 @@ func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameS
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchNode searches for nodes in the Neo4j repository
|
||||
func (n *Neo4jRepository) SearchNode(
|
||||
ctx context.Context,
|
||||
namespace types.NameSpace,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// knowledgeTagRepository is a repository for knowledge tags
|
||||
type knowledgeTagRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
@@ -17,14 +18,17 @@ func NewKnowledgeTagRepository(db *gorm.DB) interfaces.KnowledgeTagRepository {
|
||||
return &knowledgeTagRepository{db: db}
|
||||
}
|
||||
|
||||
// Create creates a new knowledge tag
|
||||
func (r *knowledgeTagRepository) Create(ctx context.Context, tag *types.KnowledgeTag) error {
|
||||
return r.db.WithContext(ctx).Create(tag).Error
|
||||
}
|
||||
|
||||
// Update updates a knowledge tag
|
||||
func (r *knowledgeTagRepository) Update(ctx context.Context, tag *types.KnowledgeTag) error {
|
||||
return r.db.WithContext(ctx).Save(tag).Error
|
||||
}
|
||||
|
||||
// GetByID gets a knowledge tag by ID
|
||||
func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, id string) (*types.KnowledgeTag, error) {
|
||||
var tag types.KnowledgeTag
|
||||
if err := r.db.WithContext(ctx).
|
||||
@@ -35,6 +39,7 @@ func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, i
|
||||
return &tag, nil
|
||||
}
|
||||
|
||||
// ListByKB lists knowledge tags by knowledge base ID
|
||||
func (r *knowledgeTagRepository) ListByKB(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
@@ -50,13 +55,14 @@ func (r *knowledgeTagRepository) ListByKB(
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// Delete deletes a knowledge tag
|
||||
func (r *knowledgeTagRepository) Delete(ctx context.Context, tenantID uint64, id string) error {
|
||||
return r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND id = ?", tenantID, id).
|
||||
Delete(&types.KnowledgeTag{}).Error
|
||||
}
|
||||
|
||||
// CountReferences returns how many knowledges and chunks reference this tag.
|
||||
// CountReferences returns the number of knowledges and chunks that reference this tag
|
||||
func (r *knowledgeTagRepository) CountReferences(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
|
||||
@@ -10,31 +10,32 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// pipelineInfo logs pipeline info level entries.
|
||||
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineInfo(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// pipelineWarn logs pipeline warning level entries.
|
||||
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineWarn(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// pipelineError logs pipeline error level entries.
|
||||
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
common.PipelineError(ctx, stage, action, fields)
|
||||
}
|
||||
|
||||
// prepareChatModel shared logic to prepare chat model and options
|
||||
// it gets the chat model and sets up the chat options based on the chat manage.
|
||||
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
|
||||
chatManage *types.ChatManage,
|
||||
) (chat.Chat, *chat.ChatOptions, error) {
|
||||
logger.Infof(ctx, "Getting chat model, model ID: %s", chatManage.ChatModelID)
|
||||
|
||||
chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chat model: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Setting up chat options")
|
||||
opt := &chat.ChatOptions{
|
||||
Temperature: chatManage.SummaryConfig.Temperature,
|
||||
TopP: chatManage.SummaryConfig.TopP,
|
||||
|
||||
@@ -94,6 +94,7 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// Extractor is a struct for extracting entities
|
||||
type Extractor struct {
|
||||
chat chat.Chat
|
||||
formater *Formater
|
||||
@@ -101,6 +102,7 @@ type Extractor struct {
|
||||
chatOpt *chat.ChatOptions
|
||||
}
|
||||
|
||||
// NewExtractor creates a new extractor
|
||||
func NewExtractor(
|
||||
chatModel chat.Chat,
|
||||
template *types.PromptTemplateStructured,
|
||||
@@ -118,6 +120,7 @@ func NewExtractor(
|
||||
}
|
||||
}
|
||||
|
||||
// Extract extracts entities from content
|
||||
func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) {
|
||||
generator := NewQAPromptGenerator(e.formater, e.template)
|
||||
|
||||
@@ -139,6 +142,7 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// RemoveUnknownRelation removes unknown relations from graph
|
||||
func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) {
|
||||
relationType := make(map[string]bool)
|
||||
for _, tag := range e.template.Tags {
|
||||
@@ -156,6 +160,7 @@ func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.Grap
|
||||
graph.Relation = relationNew
|
||||
}
|
||||
|
||||
// QAPromptGenerator is a struct for generating QA prompts
|
||||
type QAPromptGenerator struct {
|
||||
Formater *Formater
|
||||
Template *types.PromptTemplateStructured
|
||||
@@ -165,6 +170,7 @@ type QAPromptGenerator struct {
|
||||
AnswerPrefix string
|
||||
}
|
||||
|
||||
// NewQAPromptGenerator creates a new QA prompt generator
|
||||
func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator {
|
||||
return &QAPromptGenerator{
|
||||
Formater: formater,
|
||||
@@ -176,6 +182,7 @@ func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStru
|
||||
}
|
||||
}
|
||||
|
||||
// System generates a system prompt
|
||||
func (qa *QAPromptGenerator) System(ctx context.Context) string {
|
||||
promptLines := []string{}
|
||||
|
||||
@@ -205,6 +212,7 @@ func (qa *QAPromptGenerator) System(ctx context.Context) string {
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
// User generates a user prompt
|
||||
func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
|
||||
promptLines := []string{}
|
||||
promptLines = append(promptLines, qa.QuestionHeading)
|
||||
@@ -213,6 +221,7 @@ func (qa *QAPromptGenerator) User(ctx context.Context, question string) string {
|
||||
return strings.Join(promptLines, "\n")
|
||||
}
|
||||
|
||||
// Render renders a prompt
|
||||
func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message {
|
||||
return []chat.Message{
|
||||
{
|
||||
@@ -226,10 +235,13 @@ func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat
|
||||
}
|
||||
}
|
||||
|
||||
// FormatType is a type for format types
|
||||
type FormatType string
|
||||
|
||||
const (
|
||||
// FormatTypeJSON is a format type for JSON
|
||||
FormatTypeJSON FormatType = "json"
|
||||
// FormatTypeYAML is a format type for YAML
|
||||
FormatTypeYAML FormatType = "yaml"
|
||||
)
|
||||
|
||||
@@ -245,6 +257,7 @@ var _FENCE_RE = regexp.MustCompile(
|
||||
_FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END,
|
||||
)
|
||||
|
||||
// Formater is a struct for formatting entities
|
||||
type Formater struct {
|
||||
attributeSuffix string
|
||||
formatType FormatType
|
||||
@@ -256,6 +269,7 @@ type Formater struct {
|
||||
relationPrefix string
|
||||
}
|
||||
|
||||
// NewFormater creates a new formater
|
||||
func NewFormater() *Formater {
|
||||
return &Formater{
|
||||
attributeSuffix: "_attributes",
|
||||
@@ -268,6 +282,7 @@ func NewFormater() *Formater {
|
||||
}
|
||||
}
|
||||
|
||||
// formatExtraction formats extraction
|
||||
func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) {
|
||||
items := make([]map[string]interface{}, 0)
|
||||
for _, node := range nodes {
|
||||
|
||||
@@ -51,6 +51,7 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
"chunk_cnt": len(searchResult),
|
||||
})
|
||||
|
||||
// If there are no search results, return early
|
||||
if len(searchResult) == 0 {
|
||||
pipelineWarn(ctx, "Merge", "output", map[string]interface{}{
|
||||
"chunk_cnt": 0,
|
||||
@@ -212,6 +213,7 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
return nil
|
||||
}
|
||||
|
||||
// populateFAQAnswers populates FAQ answers for the search results
|
||||
func (p *PluginMerge) populateFAQAnswers(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
@@ -300,6 +302,7 @@ func (p *PluginMerge) populateFAQAnswers(
|
||||
return results
|
||||
}
|
||||
|
||||
// buildFAQAnswerContent builds the content of a FAQ answer
|
||||
func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
|
||||
if meta == nil {
|
||||
return ""
|
||||
@@ -336,6 +339,7 @@ func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
|
||||
return strings.TrimSpace(builder.String())
|
||||
}
|
||||
|
||||
// expandShortContextWithNeighbors expands the short context with neighbors
|
||||
func (p *PluginMerge) expandShortContextWithNeighbors(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
@@ -580,10 +584,12 @@ func (p *PluginMerge) expandShortContextWithNeighbors(
|
||||
return results
|
||||
}
|
||||
|
||||
// runeLen returns the length of a string in runes
|
||||
func runeLen(s string) int {
|
||||
return len([]rune(s))
|
||||
}
|
||||
|
||||
// mergeOrderedContent merges ordered content
|
||||
func mergeOrderedContent(prev, base, next string, maxLen int) string {
|
||||
content := base
|
||||
if prev != "" {
|
||||
@@ -599,7 +605,7 @@ func mergeOrderedContent(prev, base, next string, maxLen int) string {
|
||||
return content
|
||||
}
|
||||
|
||||
// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix
|
||||
// concatNoOverlap concatenates two strings, removing potential overlapping prefix/suffix
|
||||
func concatNoOverlap(a, b string) string {
|
||||
if a == "" {
|
||||
return b
|
||||
|
||||
@@ -239,6 +239,7 @@ func isInformativeToken(token string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// containsChineseCharacters checks if a string contains Chinese characters
|
||||
func containsChineseCharacters(text string) bool {
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
@@ -248,6 +249,7 @@ func containsChineseCharacters(text string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// containsLatinLetters checks if a string contains Latin letters
|
||||
func containsLatinLetters(text string) bool {
|
||||
for _, r := range text {
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
@@ -257,22 +259,26 @@ func containsLatinLetters(text string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// normalizeWhitespace normalizes whitespace in a string
|
||||
func normalizeWhitespace(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
return multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
}
|
||||
|
||||
// normalizeLatinQuery normalizes a Latin query
|
||||
func normalizeLatinQuery(text string) string {
|
||||
text = strings.ToLower(text)
|
||||
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// intentResp is a response for intent detection
|
||||
type intentResp struct {
|
||||
Intent string `json:"intent"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// detectIntentLLM detects the intent of a query using an LLM
|
||||
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
|
||||
if p.modelService == nil || chatManage.ChatModelID == "" {
|
||||
pipelineWarn(
|
||||
@@ -330,6 +336,7 @@ func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *type
|
||||
}
|
||||
}
|
||||
|
||||
// extractJSONBody extracts a JSON body from a string
|
||||
func extractJSONBody(text string) string {
|
||||
t := strings.TrimSpace(text)
|
||||
// Try fenced block first
|
||||
|
||||
@@ -176,6 +176,7 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
||||
return rankFilter
|
||||
}
|
||||
|
||||
// ensureMetadata ensures the metadata is not nil
|
||||
func ensureMetadata(m map[string]string) map[string]string {
|
||||
if m == nil {
|
||||
return make(map[string]string)
|
||||
@@ -183,6 +184,7 @@ func ensureMetadata(m map[string]string) map[string]string {
|
||||
return m
|
||||
}
|
||||
|
||||
// compositeScore calculates the composite score for a search result
|
||||
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
|
||||
sourceWeight := 1.0
|
||||
switch strings.ToLower(sr.KnowledgeSource) {
|
||||
@@ -222,6 +224,7 @@ func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatM
|
||||
return composite
|
||||
}
|
||||
|
||||
// applyMMR applies the MMR algorithm to the search results
|
||||
func applyMMR(
|
||||
ctx context.Context,
|
||||
results []*types.SearchResult,
|
||||
@@ -285,6 +288,7 @@ func applyMMR(
|
||||
return selected
|
||||
}
|
||||
|
||||
// tokenizeSimple tokenizes a text into a set of tokens
|
||||
func tokenizeSimple(text string) map[string]struct{} {
|
||||
text = strings.ToLower(text)
|
||||
fields := strings.Fields(text)
|
||||
@@ -297,6 +301,7 @@ func tokenizeSimple(text string) map[string]struct{} {
|
||||
return set
|
||||
}
|
||||
|
||||
// jaccard calculates the Jaccard similarity between two sets of tokens
|
||||
func jaccard(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 0
|
||||
@@ -314,6 +319,7 @@ func jaccard(a, b map[string]struct{}) float64 {
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
|
||||
// clampFloat clamps a float value between a minimum and maximum value
|
||||
func clampFloat(v, minV, maxV float64) float64 {
|
||||
if v < minV {
|
||||
return minV
|
||||
|
||||
@@ -16,6 +16,7 @@ type PluginSearchEntity struct {
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
}
|
||||
|
||||
// NewPluginSearchEntity creates a new plugin search entity
|
||||
func NewPluginSearchEntity(
|
||||
eventManager *EventManager,
|
||||
graphRepository interfaces.RetrieveGraphRepository,
|
||||
@@ -31,12 +32,12 @@ func NewPluginSearchEntity(
|
||||
return res
|
||||
}
|
||||
|
||||
// ActivationEvents returns the event types this plugin handles
|
||||
// ActivationEvents returns the list of event types this plugin responds to
|
||||
func (p *PluginSearchEntity) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.ENTITY_SEARCH}
|
||||
}
|
||||
|
||||
// OnEvent handles search events in the chat pipeline
|
||||
// OnEvent processes triggered events
|
||||
func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
@@ -148,6 +149,7 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// filterSeenChunk filters seen chunks from the graph
|
||||
func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, chunk := range searchResult {
|
||||
@@ -169,6 +171,7 @@ func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult [
|
||||
return chunkIDs
|
||||
}
|
||||
|
||||
// chunk2SearchResult converts a chunk to a search result
|
||||
func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult {
|
||||
return &types.SearchResult{
|
||||
ID: chunk.ID,
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// NewChunkExtractTask creates a new chunk extract task
|
||||
func NewChunkExtractTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
@@ -45,6 +46,7 @@ func NewChunkExtractTask(
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChunkExtractService is a service for extracting chunks
|
||||
type ChunkExtractService struct {
|
||||
template *types.PromptTemplateStructured
|
||||
modelService interfaces.ModelService
|
||||
@@ -53,6 +55,7 @@ type ChunkExtractService struct {
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
}
|
||||
|
||||
// NewChunkExtractService creates a new chunk extract service
|
||||
func NewChunkExtractService(
|
||||
config *config.Config,
|
||||
modelService interfaces.ModelService,
|
||||
@@ -73,6 +76,7 @@ func NewChunkExtractService(
|
||||
}
|
||||
}
|
||||
|
||||
// Extract extracts a chunk
|
||||
func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error {
|
||||
var p types.ExtractChunkPayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
@@ -137,7 +141,5 @@ func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error
|
||||
logger.Errorf(ctx, "failed to add graph: %v", err)
|
||||
return err
|
||||
}
|
||||
// gg, _ := json.Marshal(graph)
|
||||
// logger.Infof(ctx, "extracted graph: %s", string(gg))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -193,12 +193,12 @@ func (s *mcpServiceService) UpdateMCPService(ctx context.Context, service *types
|
||||
logger.GetLogger(ctx).Infof("MCP service disabled, connection closed: %s (ID: %s)", name, service.ID)
|
||||
} else if configChanged {
|
||||
s.mcpManager.CloseClient(service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed for reconnect: %s (ID: %s)", name, service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service config changed, connection closed: %s (ID: %s)", name, service.ID)
|
||||
} else if oldEnabled != existing.Enabled && existing.Enabled {
|
||||
// Service was just enabled (was disabled, now enabled)
|
||||
// Close any existing connection to ensure clean state
|
||||
s.mcpManager.CloseClient(service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed for clean state: %s (ID: %s)", name, service.ID)
|
||||
logger.GetLogger(ctx).Infof("MCP service enabled, existing connection closed: %s (ID: %s)", name, service.ID)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("MCP service updated: %s (ID: %s), enabled: %v", name, service.ID, existing.Enabled)
|
||||
|
||||
@@ -634,7 +634,8 @@ func (s *sessionService) selectChatModelID(
|
||||
return session.SummaryModelID, nil
|
||||
} else if err == nil && model != nil {
|
||||
// Session has a model but it's not Remote, we'll check knowledge bases for Remote models
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
logger.Infof(ctx, "Session has summary model %s but it's not Remote, "+
|
||||
"checking knowledge bases for Remote models", session.SummaryModelID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -952,7 +953,8 @@ func (s *sessionService) AgentQA(
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
}
|
||||
} else {
|
||||
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v", len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
|
||||
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
|
||||
}
|
||||
|
||||
summaryModelID := session.SummaryModelID
|
||||
@@ -1141,7 +1143,8 @@ func (s *sessionService) SaveWebSearchTempKBState(
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis
|
||||
// and cleans up associated knowledge base and knowledge items.
|
||||
func (s *sessionService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error {
|
||||
if s.redisClient == nil {
|
||||
return nil
|
||||
|
||||
@@ -63,6 +63,7 @@ func (p *DuckDuckGoProvider) Search(
|
||||
return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr)
|
||||
}
|
||||
|
||||
// searchHTML performs a web search using DuckDuckGo HTML endpoint
|
||||
func (p *DuckDuckGoProvider) searchHTML(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
@@ -134,6 +135,7 @@ func (p *DuckDuckGoProvider) searchHTML(
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// searchAPI performs a web search using DuckDuckGo API endpoint
|
||||
func (p *DuckDuckGoProvider) searchAPI(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
@@ -220,6 +222,7 @@ func (p *DuckDuckGoProvider) searchAPI(
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// cleanDDGURL cleans the URL from DuckDuckGo HTML endpoint
|
||||
func cleanDDGURL(urlStr string) string {
|
||||
if strings.HasPrefix(urlStr, "//duckduckgo.com/l/?uddg=") {
|
||||
trimmed := strings.TrimPrefix(urlStr, "//duckduckgo.com/l/?uddg=")
|
||||
@@ -241,6 +244,7 @@ func cleanDDGURL(urlStr string) string {
|
||||
return urlStr
|
||||
}
|
||||
|
||||
// extractTitle extracts the title from the text
|
||||
func extractTitle(text string) string {
|
||||
lines := strings.Split(text, "\n")
|
||||
if len(lines) > 0 {
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestDuckDuckGoProvider_Name(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
func TestDuckDuckGoProvider(t *testing.T) {
|
||||
// Minimal HTML page with two results, matching selectors used in searchHTML
|
||||
html := `
|
||||
<html>
|
||||
@@ -105,7 +105,7 @@ func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDuckDuckGoProvider_Search_APIFallback(t *testing.T) {
|
||||
func TestDuckDuckGoProvider_Fallback(t *testing.T) {
|
||||
// Simulate HTML returning non-OK to force API fallback, then a minimal API JSON
|
||||
apiResp := struct {
|
||||
AbstractText string `json:"AbstractText"`
|
||||
@@ -233,7 +233,7 @@ func TestDuckDuckGoProvider_Search_Real(t *testing.T) {
|
||||
|
||||
// TestDuckDuckGoProvider_Search_Real_Chinese tests the DuckDuckGo provider with Chinese query.
|
||||
// This verifies the Chinese language parameter (kl=cn-zh) works correctly.
|
||||
func TestDuckDuckGoProvider_Search_Real_Chinese(t *testing.T) {
|
||||
func TestDuckDuckGoProvider_SearchChinese(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping real DuckDuckGo integration test in short mode")
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ type FAQHandler struct {
|
||||
knowledgeService interfaces.KnowledgeService
|
||||
}
|
||||
|
||||
// NewFAQHandler creates a new FAQ handler.
|
||||
// NewFAQHandler creates a new FAQ handler
|
||||
func NewFAQHandler(knowledgeService interfaces.KnowledgeService) *FAQHandler {
|
||||
return &FAQHandler{knowledgeService: knowledgeService}
|
||||
}
|
||||
@@ -132,14 +132,17 @@ func (h *FAQHandler) UpdateEntryStatusBatch(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// faqDeleteRequest is a request for deleting FAQ entries in batch
|
||||
type faqDeleteRequest struct {
|
||||
IDs []string `json:"ids" binding:"required,min=1,dive,required"`
|
||||
}
|
||||
|
||||
// faqEntryStatusBatchRequest is a request for updating the enable status of FAQ entries in batch
|
||||
type faqEntryStatusBatchRequest struct {
|
||||
Updates map[string]bool `json:"updates" binding:"required,min=1"`
|
||||
}
|
||||
|
||||
// faqEntryTagBatchRequest is a request for updating tags for FAQ entries in batch
|
||||
type faqEntryTagBatchRequest struct {
|
||||
Updates map[string]*string `json:"updates" binding:"required,min=1"`
|
||||
}
|
||||
|
||||
+229
-178
@@ -351,12 +351,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
kbIdStr := utils.SanitizeForLog(c.Param("kbId"))
|
||||
|
||||
var req InitializationRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
logger.Error(ctx, "Failed to parse initialization request", err)
|
||||
c.Error(errors.NewBadRequestError(err.Error()))
|
||||
req, err := h.bindInitializationRequest(ctx, c)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Starting knowledge base configuration update, kbId: %s, request: %s",
|
||||
@@ -364,94 +364,152 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
utils.SanitizeForLog(utils.ToJSON(req)),
|
||||
)
|
||||
|
||||
// 获取指定知识库信息
|
||||
kb, err := h.getKnowledgeBaseForInitialization(ctx, kbIdStr)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.validateInitializationConfigs(ctx, req); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
processedModels, err := h.processInitializationModels(ctx, kb, kbIdStr, req)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
h.applyKnowledgeBaseInitialization(kb, req, processedModels)
|
||||
|
||||
if err := h.kbRepository.UpdateKnowledgeBase(ctx, kb); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"message": "知识库配置更新成功",
|
||||
"data": gin.H{
|
||||
"models": processedModels,
|
||||
"knowledge_base": kb,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) bindInitializationRequest(ctx context.Context, c *gin.Context) (*InitializationRequest, error) {
|
||||
var req InitializationRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
logger.Error(ctx, "Failed to parse initialization request", err)
|
||||
return nil, errors.NewBadRequestError(err.Error())
|
||||
}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) getKnowledgeBaseForInitialization(ctx context.Context, kbIdStr string) (*types.KnowledgeBase, error) {
|
||||
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||
c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error()))
|
||||
return
|
||||
return nil, errors.NewInternalServerError("获取知识库信息失败: " + err.Error())
|
||||
}
|
||||
|
||||
if kb == nil {
|
||||
logger.Error(ctx, "Knowledge base not found")
|
||||
c.Error(errors.NewNotFoundError("知识库不存在"))
|
||||
return
|
||||
return nil, errors.NewNotFoundError("知识库不存在")
|
||||
}
|
||||
return kb, nil
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) validateInitializationConfigs(ctx context.Context, req *InitializationRequest) error {
|
||||
if err := h.validateMultimodalConfig(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRerankConfig(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateNodeExtractConfig(ctx, req)
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) validateMultimodalConfig(ctx context.Context, req *InitializationRequest) error {
|
||||
if !req.Multimodal.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 验证多模态配置(如果启用)
|
||||
if req.Multimodal.Enabled {
|
||||
storageType := strings.ToLower(req.Multimodal.StorageType)
|
||||
if req.Multimodal.VLM == nil {
|
||||
logger.Error(ctx, "Multimodal enabled but missing VLM configuration")
|
||||
c.Error(errors.NewBadRequestError("启用多模态时需要配置VLM信息"))
|
||||
return
|
||||
}
|
||||
if req.Multimodal.VLM.InterfaceType == "ollama" {
|
||||
req.Multimodal.VLM.BaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1"
|
||||
}
|
||||
if req.Multimodal.VLM.ModelName == "" || req.Multimodal.VLM.BaseURL == "" {
|
||||
logger.Error(ctx, "VLM configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("VLM配置不完整"))
|
||||
return
|
||||
}
|
||||
switch storageType {
|
||||
case "cos":
|
||||
if req.Multimodal.COS == nil || req.Multimodal.COS.SecretID == "" || req.Multimodal.COS.SecretKey == "" ||
|
||||
req.Multimodal.COS.Region == "" || req.Multimodal.COS.BucketName == "" ||
|
||||
req.Multimodal.COS.AppID == "" {
|
||||
logger.Error(ctx, "COS configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("COS配置不完整"))
|
||||
return
|
||||
}
|
||||
case "minio":
|
||||
if req.Multimodal.Minio == nil || req.Multimodal.Minio.BucketName == "" ||
|
||||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" || os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" {
|
||||
logger.Error(ctx, "MinIO configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("MinIO配置不完整"))
|
||||
return
|
||||
}
|
||||
}
|
||||
storageType := strings.ToLower(req.Multimodal.StorageType)
|
||||
if req.Multimodal.VLM == nil {
|
||||
logger.Error(ctx, "Multimodal enabled but missing VLM configuration")
|
||||
return errors.NewBadRequestError("启用多模态时需要配置VLM信息")
|
||||
}
|
||||
if req.Multimodal.VLM.InterfaceType == "ollama" {
|
||||
req.Multimodal.VLM.BaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1"
|
||||
}
|
||||
if req.Multimodal.VLM.ModelName == "" || req.Multimodal.VLM.BaseURL == "" {
|
||||
logger.Error(ctx, "VLM configuration incomplete")
|
||||
return errors.NewBadRequestError("VLM配置不完整")
|
||||
}
|
||||
|
||||
// 验证Rerank配置(如果启用)
|
||||
if req.Rerank.Enabled {
|
||||
if req.Rerank.ModelName == "" || req.Rerank.BaseURL == "" {
|
||||
logger.Error(ctx, "Rerank configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("Rerank配置不完整"))
|
||||
return
|
||||
switch storageType {
|
||||
case "cos":
|
||||
if req.Multimodal.COS == nil || req.Multimodal.COS.SecretID == "" || req.Multimodal.COS.SecretKey == "" ||
|
||||
req.Multimodal.COS.Region == "" || req.Multimodal.COS.BucketName == "" ||
|
||||
req.Multimodal.COS.AppID == "" {
|
||||
logger.Error(ctx, "COS configuration incomplete")
|
||||
return errors.NewBadRequestError("COS配置不完整")
|
||||
}
|
||||
case "minio":
|
||||
if req.Multimodal.Minio == nil || req.Multimodal.Minio.BucketName == "" ||
|
||||
os.Getenv("MINIO_ACCESS_KEY_ID") == "" || os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" {
|
||||
logger.Error(ctx, "MinIO configuration incomplete")
|
||||
return errors.NewBadRequestError("MinIO配置不完整")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 验证Node Extractor配置(如果启用)
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" && req.NodeExtract.Enabled {
|
||||
func validateRerankConfig(ctx context.Context, req *InitializationRequest) error {
|
||||
if !req.Rerank.Enabled {
|
||||
return nil
|
||||
}
|
||||
if req.Rerank.ModelName == "" || req.Rerank.BaseURL == "" {
|
||||
logger.Error(ctx, "Rerank configuration incomplete")
|
||||
return errors.NewBadRequestError("Rerank配置不完整")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateNodeExtractConfig(ctx context.Context, req *InitializationRequest) error {
|
||||
if !req.NodeExtract.Enabled {
|
||||
return nil
|
||||
}
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE"))
|
||||
return
|
||||
return errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE")
|
||||
}
|
||||
if req.NodeExtract.Enabled {
|
||||
if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("Node Extractor配置不完整"))
|
||||
return
|
||||
}
|
||||
if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
c.Error(errors.NewBadRequestError("请先提取实体和关系"))
|
||||
return
|
||||
}
|
||||
if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
return errors.NewBadRequestError("Node Extractor配置不完整")
|
||||
}
|
||||
if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 {
|
||||
logger.Error(ctx, "Node Extractor configuration incomplete")
|
||||
return errors.NewBadRequestError("请先提取实体和关系")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 处理模型创建/更新
|
||||
modelsToProcess := []struct {
|
||||
modelType types.ModelType
|
||||
name string
|
||||
source types.ModelSource
|
||||
description string
|
||||
baseURL string
|
||||
apiKey string
|
||||
dimension int
|
||||
interfaceType string
|
||||
}{
|
||||
type modelDescriptor struct {
|
||||
modelType types.ModelType
|
||||
name string
|
||||
source types.ModelSource
|
||||
description string
|
||||
baseURL string
|
||||
apiKey string
|
||||
dimension int
|
||||
interfaceType string
|
||||
}
|
||||
|
||||
func buildModelDescriptors(req *InitializationRequest) []modelDescriptor {
|
||||
descriptors := []modelDescriptor{
|
||||
{
|
||||
modelType: types.ModelTypeKnowledgeQA,
|
||||
name: utils.SanitizeForLog(req.LLM.ModelName),
|
||||
@@ -471,18 +529,8 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
},
|
||||
}
|
||||
|
||||
// 如果启用Rerank,添加Rerank模型
|
||||
if req.Rerank.Enabled {
|
||||
modelsToProcess = append(modelsToProcess, struct {
|
||||
modelType types.ModelType
|
||||
name string
|
||||
source types.ModelSource
|
||||
description string
|
||||
baseURL string
|
||||
apiKey string
|
||||
dimension int
|
||||
interfaceType string
|
||||
}{
|
||||
descriptors = append(descriptors, modelDescriptor{
|
||||
modelType: types.ModelTypeRerank,
|
||||
name: utils.SanitizeForLog(req.Rerank.ModelName),
|
||||
source: types.ModelSourceRemote,
|
||||
@@ -492,18 +540,8 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// 如果启用多模态,添加VLM模型
|
||||
if req.Multimodal.Enabled && req.Multimodal.VLM != nil {
|
||||
modelsToProcess = append(modelsToProcess, struct {
|
||||
modelType types.ModelType
|
||||
name string
|
||||
source types.ModelSource
|
||||
description string
|
||||
baseURL string
|
||||
apiKey string
|
||||
dimension int
|
||||
interfaceType string
|
||||
}{
|
||||
descriptors = append(descriptors, modelDescriptor{
|
||||
modelType: types.ModelTypeVLLM,
|
||||
name: utils.SanitizeForLog(req.Multimodal.VLM.ModelName),
|
||||
source: types.ModelSourceRemote,
|
||||
@@ -514,113 +552,116 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// 处理模型
|
||||
return descriptors
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) processInitializationModels(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
kbIdStr string,
|
||||
req *InitializationRequest,
|
||||
) ([]*types.Model, error) {
|
||||
descriptors := buildModelDescriptors(req)
|
||||
var processedModels []*types.Model
|
||||
for _, modelInfo := range modelsToProcess {
|
||||
model := &types.Model{
|
||||
Type: modelInfo.modelType,
|
||||
Name: modelInfo.name,
|
||||
Source: modelInfo.source,
|
||||
Description: modelInfo.description,
|
||||
Parameters: types.ModelParameters{
|
||||
BaseURL: modelInfo.baseURL,
|
||||
APIKey: modelInfo.apiKey,
|
||||
InterfaceType: modelInfo.interfaceType,
|
||||
},
|
||||
IsDefault: false,
|
||||
Status: types.ModelStatusActive,
|
||||
}
|
||||
|
||||
if modelInfo.modelType == types.ModelTypeEmbedding {
|
||||
model.Parameters.EmbeddingParameters = types.EmbeddingParameters{
|
||||
Dimension: modelInfo.dimension,
|
||||
}
|
||||
}
|
||||
for _, descriptor := range descriptors {
|
||||
model := descriptor.toModel()
|
||||
existingModelID := h.findExistingModelID(kb, descriptor.modelType)
|
||||
|
||||
// 根据模型类型从知识库获取已有的模型ID
|
||||
var existingModelID string
|
||||
switch modelInfo.modelType {
|
||||
case types.ModelTypeEmbedding:
|
||||
existingModelID = kb.EmbeddingModelID
|
||||
case types.ModelTypeKnowledgeQA:
|
||||
existingModelID = kb.SummaryModelID
|
||||
case types.ModelTypeVLLM:
|
||||
existingModelID = kb.VLMConfig.ModelID
|
||||
}
|
||||
|
||||
// 如果知识库中已经有对应类型的模型ID,尝试获取并更新
|
||||
var existingModel *types.Model
|
||||
if existingModelID != "" {
|
||||
var err error
|
||||
existingModel, err = h.modelService.GetModelByID(ctx, existingModelID)
|
||||
if err != nil {
|
||||
// 模型不存在或获取失败,记录日志但继续创建新模型
|
||||
logger.Warnf(ctx, "Failed to get existing model %s: %v, will create new one", existingModelID, err)
|
||||
existingModel = nil
|
||||
}
|
||||
}
|
||||
|
||||
if existingModel != nil {
|
||||
// 更新现有模型
|
||||
existingModel.Name = model.Name
|
||||
existingModel.Source = model.Source
|
||||
existingModel.Description = model.Description
|
||||
existingModel.Parameters = model.Parameters
|
||||
existingModel.UpdatedAt = time.Now()
|
||||
|
||||
err = h.modelService.UpdateModel(ctx, existingModel)
|
||||
if err != nil {
|
||||
if err := h.modelService.UpdateModel(ctx, existingModel); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"kb_id": kbIdStr,
|
||||
})
|
||||
c.Error(errors.NewInternalServerError("更新模型失败: " + err.Error()))
|
||||
return
|
||||
return nil, errors.NewInternalServerError("更新模型失败: " + err.Error())
|
||||
}
|
||||
processedModels = append(processedModels, existingModel)
|
||||
} else {
|
||||
// 创建新模型
|
||||
err = h.modelService.CreateModel(ctx, model)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"kb_id": kbIdStr,
|
||||
})
|
||||
c.Error(errors.NewInternalServerError("创建模型失败: " + err.Error()))
|
||||
return
|
||||
}
|
||||
processedModels = append(processedModels, model)
|
||||
}
|
||||
}
|
||||
|
||||
// 找到模型ID
|
||||
var embeddingModelID, llmModelID, vlmModelID string
|
||||
for _, model := range processedModels {
|
||||
if model == nil {
|
||||
continue
|
||||
}
|
||||
if model.Type == types.ModelTypeEmbedding {
|
||||
embeddingModelID = model.ID
|
||||
|
||||
if err := h.modelService.CreateModel(ctx, model); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"model_id": model.ID,
|
||||
"kb_id": kbIdStr,
|
||||
})
|
||||
return nil, errors.NewInternalServerError("创建模型失败: " + err.Error())
|
||||
}
|
||||
if model.Type == types.ModelTypeKnowledgeQA {
|
||||
llmModelID = model.ID
|
||||
}
|
||||
if model.Type == types.ModelTypeVLLM {
|
||||
vlmModelID = model.ID
|
||||
processedModels = append(processedModels, model)
|
||||
}
|
||||
|
||||
return processedModels, nil
|
||||
}
|
||||
|
||||
func (descriptor modelDescriptor) toModel() *types.Model {
|
||||
model := &types.Model{
|
||||
Type: descriptor.modelType,
|
||||
Name: descriptor.name,
|
||||
Source: descriptor.source,
|
||||
Description: descriptor.description,
|
||||
Parameters: types.ModelParameters{
|
||||
BaseURL: descriptor.baseURL,
|
||||
APIKey: descriptor.apiKey,
|
||||
InterfaceType: descriptor.interfaceType,
|
||||
},
|
||||
IsDefault: false,
|
||||
Status: types.ModelStatusActive,
|
||||
}
|
||||
|
||||
if descriptor.modelType == types.ModelTypeEmbedding {
|
||||
model.Parameters.EmbeddingParameters = types.EmbeddingParameters{
|
||||
Dimension: descriptor.dimension,
|
||||
}
|
||||
}
|
||||
|
||||
// 更新知识库配置
|
||||
return model
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) findExistingModelID(kb *types.KnowledgeBase, modelType types.ModelType) string {
|
||||
switch modelType {
|
||||
case types.ModelTypeEmbedding:
|
||||
return kb.EmbeddingModelID
|
||||
case types.ModelTypeKnowledgeQA:
|
||||
return kb.SummaryModelID
|
||||
case types.ModelTypeVLLM:
|
||||
return kb.VLMConfig.ModelID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (h *InitializationHandler) applyKnowledgeBaseInitialization(
|
||||
kb *types.KnowledgeBase,
|
||||
req *InitializationRequest,
|
||||
processedModels []*types.Model,
|
||||
) {
|
||||
embeddingModelID, llmModelID, vlmModelID := extractModelIDs(processedModels)
|
||||
|
||||
kb.SummaryModelID = llmModelID
|
||||
kb.EmbeddingModelID = embeddingModelID
|
||||
|
||||
// 更新文档分割配置
|
||||
kb.ChunkingConfig = types.ChunkingConfig{
|
||||
ChunkSize: req.DocumentSplitting.ChunkSize,
|
||||
ChunkOverlap: req.DocumentSplitting.ChunkOverlap,
|
||||
Separators: req.DocumentSplitting.Separators,
|
||||
}
|
||||
|
||||
// 更新多模态配置
|
||||
if req.Multimodal.Enabled {
|
||||
kb.VLMConfig = types.VLMConfig{
|
||||
Enabled: req.Multimodal.Enabled,
|
||||
@@ -677,22 +718,23 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": utils.SanitizeForLog(kbIdStr)})
|
||||
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
|
||||
return
|
||||
func extractModelIDs(processedModels []*types.Model) (embeddingModelID, llmModelID, vlmModelID string) {
|
||||
for _, model := range processedModels {
|
||||
if model == nil {
|
||||
continue
|
||||
}
|
||||
switch model.Type {
|
||||
case types.ModelTypeEmbedding:
|
||||
embeddingModelID = model.ID
|
||||
case types.ModelTypeKnowledgeQA:
|
||||
llmModelID = model.ID
|
||||
case types.ModelTypeVLLM:
|
||||
vlmModelID = model.ID
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"message": "知识库配置更新成功",
|
||||
"data": gin.H{
|
||||
"models": processedModels,
|
||||
"knowledge_base": kb,
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// CheckOllamaStatus 检查Ollama服务状态
|
||||
@@ -1787,6 +1829,7 @@ type TextRelationExtractionResponse struct {
|
||||
Relations []*types.GraphRelation `json:"relations"`
|
||||
}
|
||||
|
||||
// ExtractTextRelations extracts text relations from text
|
||||
func (h *InitializationHandler) ExtractTextRelations(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
@@ -1869,15 +1912,18 @@ func (h *InitializationHandler) extractRelationsFromText(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// FabriTextRequest is a request for generating example text
|
||||
type FabriTextRequest struct {
|
||||
Tags []string `json:"tags"`
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
// FabriTextResponse is a response for generating example text
|
||||
type FabriTextResponse struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// FabriText generates example text
|
||||
func (h *InitializationHandler) FabriText(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
@@ -1901,6 +1947,7 @@ func (h *InitializationHandler) FabriText(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// fabriText generates example text
|
||||
func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, llm LLMConfig) (string, error) {
|
||||
chatModel, err := chat.NewChat(&chat.ChatConfig{
|
||||
ModelID: "initialization",
|
||||
@@ -1935,10 +1982,12 @@ func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, ll
|
||||
return result.Content, nil
|
||||
}
|
||||
|
||||
// FabriTagRequest is a request for generating tags
|
||||
type FabriTagRequest struct {
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
// FabriTagResponse is a response for generating tags
|
||||
type FabriTagResponse struct {
|
||||
Tags []string `json:"tags"`
|
||||
}
|
||||
@@ -1947,6 +1996,7 @@ var tagOptions = []string{
|
||||
"内容", "文化", "人物", "事件", "时间", "地点", "作品", "作者", "关系", "属性",
|
||||
}
|
||||
|
||||
// FabriTag generates tags
|
||||
func (h *InitializationHandler) FabriTag(c *gin.Context) {
|
||||
tagRandom := RandomSelect(tagOptions, rand.Intn(len(tagOptions)-1)+1)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
@@ -1955,6 +2005,7 @@ func (h *InitializationHandler) FabriTag(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// RandomSelect selects random strings
|
||||
func RandomSelect(strs []string, n int) []string {
|
||||
if n <= 0 {
|
||||
return []string{}
|
||||
|
||||
@@ -159,7 +159,7 @@ func (h *Handler) AgentQA(c *gin.Context) {
|
||||
if requestJSON, err := json.Marshal(request); err == nil {
|
||||
logger.Infof(ctx, "Agent QA request, request: %s", secutils.SanitizeForLog(string(requestJSON)))
|
||||
} else {
|
||||
logger.Warnf(ctx, "Agent QA request received but failed to marshal for logging: %s", secutils.SanitizeForLog(err.Error()))
|
||||
logger.Warnf(ctx, "failed to marshal for logging: %s", secutils.SanitizeForLog(err.Error()))
|
||||
}
|
||||
|
||||
// Validate query content
|
||||
|
||||
@@ -431,7 +431,7 @@ func (h *TenantHandler) UpdateTenantKV(c *gin.Context) {
|
||||
h.updateTenantWebSearchConfigInternal(c)
|
||||
return
|
||||
case "conversation-config":
|
||||
h.updateTenantConversationConfigInternal(c)
|
||||
h.updateTenantConversationInternal(c)
|
||||
return
|
||||
default:
|
||||
logger.Info(ctx, "KV key not supported", "key", key)
|
||||
@@ -664,9 +664,9 @@ func (h *TenantHandler) GetTenantConversationConfig(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// updateTenantConversationConfigInternal updates the conversation configuration for a tenant
|
||||
// updateTenantConversationInternal updates the conversation configuration for a tenant
|
||||
// This sets the global conversation configuration for normal mode sessions in this tenant
|
||||
func (h *TenantHandler) updateTenantConversationConfigInternal(c *gin.Context) {
|
||||
func (h *TenantHandler) updateTenantConversationInternal(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
logger.Info(ctx, "Start updating tenant conversation config")
|
||||
|
||||
|
||||
@@ -43,7 +43,8 @@ type ChunkRepository interface {
|
||||
CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error)
|
||||
// DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range
|
||||
DeleteUnindexedChunks(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error)
|
||||
// ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID (only ID and ContentHash fields for efficiency)
|
||||
// ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID
|
||||
// only ID and ContentHash fields for efficiency
|
||||
ListAllFAQChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// Extracter is a interface for extracting entities
|
||||
type Extracter interface {
|
||||
// Extract extracts entities from a task
|
||||
Extract(ctx context.Context, t *asynq.Task) error
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ type ModelRepository interface {
|
||||
Update(ctx context.Context, model *types.Model) error
|
||||
// Delete deletes a model
|
||||
Delete(ctx context.Context, tenantID uint64, id string) error
|
||||
// ClearDefaultByType clears the default flag for all models of a specific type, optionally excluding a specific model ID
|
||||
// ClearDefaultByType clears the default flag for all models of a specific type
|
||||
// optionally excluding a specific model ID.
|
||||
ClearDefaultByType(ctx context.Context, tenantID uint, modelType types.ModelType, excludeID string) error
|
||||
}
|
||||
|
||||
@@ -6,8 +6,12 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// RetrieveGraphRepository is a repository for retrieving graphs
|
||||
type RetrieveGraphRepository interface {
|
||||
// AddGraph adds a graph to the repository
|
||||
AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error
|
||||
// DelGraph deletes a graph from the repository
|
||||
DelGraph(ctx context.Context, namespace []types.NameSpace) error
|
||||
// SearchNode searches for nodes in the repository
|
||||
SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,8 @@ type SessionService interface {
|
||||
// webSearchEnabled: whether to enable web search to supplement knowledge base results
|
||||
// Events are emitted through eventBus (references, answer chunks, completion)
|
||||
KnowledgeQA(ctx context.Context,
|
||||
session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus,
|
||||
session *types.Session, query string, knowledgeBaseIDs []string,
|
||||
assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus,
|
||||
) error
|
||||
// KnowledgeQAByEvent performs knowledge-based question answering by event
|
||||
KnowledgeQAByEvent(ctx context.Context, chatManage *types.ChatManage, eventList []types.EventType) error
|
||||
@@ -62,7 +63,8 @@ type SessionService interface {
|
||||
seenURLs map[string]bool,
|
||||
knowledgeIDs []string,
|
||||
)
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis
|
||||
// and cleans up associated knowledge base and knowledge items
|
||||
DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error
|
||||
}
|
||||
|
||||
|
||||
@@ -8,12 +8,18 @@ import (
|
||||
|
||||
// WebSearchProvider defines the interface for web search providers
|
||||
type WebSearchProvider interface {
|
||||
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
|
||||
// Name returns the name of the provider
|
||||
Name() string
|
||||
// Search performs a web search
|
||||
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
|
||||
}
|
||||
|
||||
// WebSearchService defines the interface for web search services
|
||||
type WebSearchService interface {
|
||||
// Search performs a web search
|
||||
Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error)
|
||||
// CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base
|
||||
// The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering.
|
||||
CompressWithRAG(ctx context.Context, sessionID string, tempKBID string, questions []string,
|
||||
webSearchResults []*types.WebSearchResult, cfg *types.WebSearchConfig,
|
||||
kbSvc KnowledgeBaseService, knowSvc KnowledgeService,
|
||||
|
||||
@@ -2,6 +2,7 @@ package utils
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// ToJSON converts a value to a JSON string
|
||||
func ToJSON(v interface{}) string {
|
||||
json, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user