feat: Integrate LLM-based reranking in KnowledgeSearchTool
- Added optional chat model to KnowledgeSearchTool for enhanced reranking using LLM prompt scoring. - Updated rerankResults method to prioritize LLM-based reranking over traditional rerank model. - Implemented new functions for LLM scoring and parsing scores from responses, improving search result relevance. - Enhanced migration script to handle database versioning and dirty state checks more robustly.
This commit is contained in:
@@ -5,9 +5,12 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -27,6 +30,7 @@ type KnowledgeSearchTool struct {
|
||||
tenantID uint
|
||||
allowedKBs []string
|
||||
rerankModel rerank.Reranker
|
||||
chatModel chat.Chat // Optional chat model for LLM-based reranking
|
||||
}
|
||||
|
||||
// NewKnowledgeSearchTool creates a new knowledge search tool
|
||||
@@ -35,6 +39,7 @@ func NewKnowledgeSearchTool(
|
||||
tenantID uint,
|
||||
allowedKBs []string,
|
||||
rerankModel rerank.Reranker,
|
||||
chatModel chat.Chat,
|
||||
) *KnowledgeSearchTool {
|
||||
description := `Search within knowledge bases with flexible query modes. Unified tool that supports both targeted and broad searches.
|
||||
|
||||
@@ -108,6 +113,7 @@ func NewKnowledgeSearchTool(
|
||||
tenantID: tenantID,
|
||||
allowedKBs: allowedKBs,
|
||||
rerankModel: rerankModel,
|
||||
chatModel: chatModel,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,7 +352,29 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
|
||||
len(filteredResults), len(allResults))
|
||||
|
||||
// Apply ReRank if model is configured
|
||||
if t.rerankModel != nil && len(filteredResults) > 0 {
|
||||
// Prefer chatModel (LLM-based reranking) over rerankModel if both are available
|
||||
if t.chatModel != nil && len(filteredResults) > 0 {
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying LLM-based rerank with model: %s, input: %d results",
|
||||
t.chatModel.GetModelName(), len(filteredResults))
|
||||
rerankQuery := singleQuery
|
||||
if rerankQuery == "" && len(vectorQueries) > 0 {
|
||||
rerankQuery = vectorQueries[0] // Use first vector query as rerank query
|
||||
} else if rerankQuery == "" && len(keywordQueries) > 0 {
|
||||
rerankQuery = keywordQueries[0] // Use first keyword query as fallback
|
||||
}
|
||||
|
||||
if rerankQuery != "" {
|
||||
logger.Debugf(ctx, "[Tool][KnowledgeSearch] Rerank query: %s", rerankQuery)
|
||||
rerankedResults, err := t.rerankResults(ctx, rerankQuery, filteredResults)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] LLM rerank failed, using original results: %v", err)
|
||||
} else {
|
||||
filteredResults = rerankedResults
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM rerank completed successfully: %d results",
|
||||
len(filteredResults))
|
||||
}
|
||||
}
|
||||
} else if t.rerankModel != nil && len(filteredResults) > 0 {
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank with model: %s, input: %d results",
|
||||
t.rerankModel.GetModelName(), len(filteredResults))
|
||||
rerankQuery := singleQuery
|
||||
@@ -560,11 +588,180 @@ func (t *KnowledgeSearchTool) filterByThreshold(
|
||||
return filtered
|
||||
}
|
||||
|
||||
// rerankResults applies reranking to search results using the configured rerank model
|
||||
// rerankResults applies reranking to search results using LLM prompt scoring or rerank model
|
||||
func (t *KnowledgeSearchTool) rerankResults(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
results []*searchResultWithMeta,
|
||||
) ([]*searchResultWithMeta, error) {
|
||||
// If chatModel is available, use LLM-based prompt scoring
|
||||
if t.chatModel != nil {
|
||||
return t.rerankWithLLM(ctx, query, results)
|
||||
}
|
||||
|
||||
// Fallback to rerank model if available
|
||||
if t.rerankModel != nil {
|
||||
return t.rerankWithModel(ctx, query, results)
|
||||
}
|
||||
|
||||
// No reranking available, return original results
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// rerankWithLLM uses LLM prompt to score and rerank search results
|
||||
func (t *KnowledgeSearchTool) rerankWithLLM(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
results []*searchResultWithMeta,
|
||||
) ([]*searchResultWithMeta, error) {
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Using LLM for reranking %d results", len(results))
|
||||
|
||||
// Build prompt with query and all passages
|
||||
var passagesBuilder strings.Builder
|
||||
for i, result := range results {
|
||||
passagesBuilder.WriteString(fmt.Sprintf("Passage %d:\n%s\n\n", i+1, result.Content))
|
||||
}
|
||||
|
||||
prompt := fmt.Sprintf(`你是一个相关性评分专家。请根据查询问题,对以下每个段落进行相关性评分。
|
||||
|
||||
查询问题:%s
|
||||
|
||||
评分要求:
|
||||
1. 评分范围:0.0 到 1.0 之间的小数
|
||||
2. 1.0 表示完全相关,0.0 表示完全不相关
|
||||
3. 考虑语义相关性、信息完整性、准确性等因素
|
||||
4. 必须为每个段落返回一个评分
|
||||
5. 输出格式:每行一个评分,格式为 "Passage N: X.XX"(N 是段落编号,X.XX 是 0.0-1.0 之间的分数)
|
||||
|
||||
段落列表:
|
||||
%s
|
||||
|
||||
请严格按照以下格式返回每个段落的评分(每行一个,共 %d 个):
|
||||
Passage 1: X.XX
|
||||
Passage 2: X.XX
|
||||
...
|
||||
Passage %d: X.XX`, query, passagesBuilder.String(), len(results), len(results))
|
||||
|
||||
messages := []chat.Message{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "你是一个专业的文本相关性评分专家,能够准确评估文本与查询问题的相关性。",
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: prompt,
|
||||
},
|
||||
}
|
||||
|
||||
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
|
||||
Temperature: 0.1, // Low temperature for consistent scoring
|
||||
MaxTokens: 1024,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LLM rerank call failed: %w", err)
|
||||
}
|
||||
|
||||
// Parse scores from response
|
||||
scores, err := t.parseScoresFromResponse(response.Content, len(results))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to parse LLM scores: %v, using original scores", err)
|
||||
return results, nil // Return original results if parsing fails
|
||||
}
|
||||
|
||||
// Create reranked results with new scores
|
||||
reranked := make([]*searchResultWithMeta, 0, len(results))
|
||||
for i, result := range results {
|
||||
newResult := *result
|
||||
if i < len(scores) {
|
||||
newResult.Score = scores[i]
|
||||
}
|
||||
reranked = append(reranked, &newResult)
|
||||
}
|
||||
|
||||
// Sort by new scores (descending)
|
||||
sort.Slice(reranked, func(i, j int) bool {
|
||||
return reranked[i].Score > reranked[j].Score
|
||||
})
|
||||
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d results from %d original results", len(reranked), len(results))
|
||||
return reranked, nil
|
||||
}
|
||||
|
||||
// parseScoresFromResponse parses scores from LLM response text
|
||||
func (t *KnowledgeSearchTool) parseScoresFromResponse(responseText string, expectedCount int) ([]float64, error) {
|
||||
lines := strings.Split(strings.TrimSpace(responseText), "\n")
|
||||
scores := make([]float64, 0, expectedCount)
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to extract score from various formats:
|
||||
// "Passage 1: 0.85"
|
||||
// "1: 0.85"
|
||||
// "0.85"
|
||||
// etc.
|
||||
parts := strings.Split(line, ":")
|
||||
var scoreStr string
|
||||
if len(parts) >= 2 {
|
||||
scoreStr = strings.TrimSpace(parts[len(parts)-1])
|
||||
} else {
|
||||
scoreStr = strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
// Remove any non-numeric characters except decimal point
|
||||
scoreStr = strings.TrimFunc(scoreStr, func(r rune) bool {
|
||||
return (r < '0' || r > '9') && r != '.'
|
||||
})
|
||||
|
||||
if scoreStr == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
score, err := strconv.ParseFloat(scoreStr, 64)
|
||||
if err != nil {
|
||||
continue // Skip invalid scores
|
||||
}
|
||||
|
||||
// Clamp score to [0.0, 1.0]
|
||||
if score < 0.0 {
|
||||
score = 0.0
|
||||
}
|
||||
if score > 1.0 {
|
||||
score = 1.0
|
||||
}
|
||||
|
||||
scores = append(scores, score)
|
||||
}
|
||||
|
||||
if len(scores) == 0 {
|
||||
return nil, fmt.Errorf("no valid scores found in response")
|
||||
}
|
||||
|
||||
// If we got fewer scores than expected, pad with last score or 0.5
|
||||
for len(scores) < expectedCount {
|
||||
if len(scores) > 0 {
|
||||
scores = append(scores, scores[len(scores)-1])
|
||||
} else {
|
||||
scores = append(scores, 0.5)
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate if we got more scores than expected
|
||||
if len(scores) > expectedCount {
|
||||
scores = scores[:expectedCount]
|
||||
}
|
||||
|
||||
return scores, nil
|
||||
}
|
||||
|
||||
// rerankWithModel uses the rerank model for reranking (fallback)
|
||||
func (t *KnowledgeSearchTool) rerankWithModel(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
results []*searchResultWithMeta,
|
||||
) ([]*searchResultWithMeta, error) {
|
||||
// Prepare passages for reranking
|
||||
passages := make([]string, len(results))
|
||||
@@ -589,7 +786,7 @@ func (t *KnowledgeSearchTool) rerankResults(
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Reranked %d results from %d original results", len(reranked), len(results))
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Reranked %d results from %d original results", len(reranked), len(results))
|
||||
return reranked, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -208,7 +208,7 @@ func (s *agentService) registerTools(
|
||||
case "todo_write":
|
||||
registry.RegisterTool(tools.NewTodoWriteTool())
|
||||
case "knowledge_search":
|
||||
registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel))
|
||||
registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel, chatModel))
|
||||
case "get_related_chunks":
|
||||
registry.RegisterTool(tools.NewGetRelatedChunksTool(s.chunkService, s.knowledgeBaseService))
|
||||
case "query_knowledge_graph":
|
||||
|
||||
@@ -23,11 +23,59 @@ func RunMigrations(dsn string) error {
|
||||
}
|
||||
defer m.Close()
|
||||
|
||||
// Get current version before migration
|
||||
oldVersion, _, _ := m.Version()
|
||||
// Check current version and dirty state before migration
|
||||
oldVersion, oldDirty, versionErr := m.Version()
|
||||
if versionErr != nil && versionErr != migrate.ErrNilVersion {
|
||||
// If we can't get version and it's not because database is empty, return error
|
||||
return fmt.Errorf("failed to get migration version: %w", versionErr)
|
||||
}
|
||||
|
||||
// If database is in dirty state, return detailed error with fix instructions
|
||||
if oldDirty {
|
||||
// Calculate the version to force to (usually the previous version)
|
||||
forceVersion := int(oldVersion) - 1
|
||||
if oldVersion == 0 || forceVersion < 0 {
|
||||
forceVersion = 0
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"database is in dirty state at version %d. This usually means a migration failed partway through. "+
|
||||
"To fix this:\n"+
|
||||
"1. Check if the migration partially applied changes and manually fix if needed\n"+
|
||||
"2. Use the force command to set the version to the last successful migration (usually %d):\n"+
|
||||
" ./scripts/migrate.sh force %d\n"+
|
||||
" Or if using make: make migrate-force version=%d\n"+
|
||||
"3. After fixing, restart the application to retry the migration",
|
||||
oldVersion,
|
||||
forceVersion,
|
||||
forceVersion,
|
||||
forceVersion,
|
||||
)
|
||||
}
|
||||
|
||||
// Run all pending migrations
|
||||
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
|
||||
// Check if error is due to dirty state (in case it became dirty during migration)
|
||||
currentVersion, currentDirty, versionCheckErr := m.Version()
|
||||
if versionCheckErr == nil && currentDirty {
|
||||
// Calculate the version to force to (usually the previous version)
|
||||
forceVersion := currentVersion - 1
|
||||
if currentVersion == 0 {
|
||||
forceVersion = 0
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"migration failed and database is now in dirty state at version %d. "+
|
||||
"To fix this:\n"+
|
||||
"1. Check if the migration partially applied changes and manually fix if needed\n"+
|
||||
"2. Use the force command to set the version to the last successful migration (usually %d):\n"+
|
||||
" ./scripts/migrate.sh force %d\n"+
|
||||
" Or if using make: make migrate-force version=%d\n"+
|
||||
"3. After fixing, restart the application to retry the migration",
|
||||
currentVersion,
|
||||
forceVersion,
|
||||
forceVersion,
|
||||
forceVersion,
|
||||
)
|
||||
}
|
||||
return fmt.Errorf("failed to run migrations: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,11 +3,20 @@
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- Add web_search_config column to tenants table
|
||||
ALTER TABLE tenants
|
||||
ADD COLUMN web_search_config JSONB DEFAULT NULL;
|
||||
-- Add web_search_config column to tenants table (if not exists)
|
||||
-- Note: PostgreSQL 9.6+ supports IF NOT EXISTS
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'tenants' AND column_name = 'web_search_config'
|
||||
) THEN
|
||||
ALTER TABLE tenants
|
||||
ADD COLUMN web_search_config JSONB DEFAULT NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Add comment
|
||||
-- Add or update comment
|
||||
COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant';
|
||||
|
||||
COMMIT;
|
||||
|
||||
+39
-1
@@ -1,6 +1,18 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Get the script directory and project root
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
|
||||
|
||||
# Load .env file if it exists (for development mode)
|
||||
if [ -f "$PROJECT_ROOT/.env" ]; then
|
||||
echo "Loading .env file from $PROJECT_ROOT/.env"
|
||||
set -a
|
||||
source "$PROJECT_ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Database connection details (can be overridden by environment variables)
|
||||
DB_HOST=${DB_HOST:-localhost}
|
||||
DB_PORT=${DB_PORT:-5432}
|
||||
@@ -19,7 +31,33 @@ if ! command -v migrate &> /dev/null; then
|
||||
fi
|
||||
|
||||
# Construct the database URL
|
||||
DB_URL="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
|
||||
# If DB_URL is already set in .env, use it but ensure sslmode=disable is set
|
||||
# Otherwise, construct it from individual components
|
||||
if [ -n "$DB_URL" ]; then
|
||||
# If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified)
|
||||
if [[ "$DB_URL" != *"sslmode="* ]]; then
|
||||
# Add sslmode=disable if not present
|
||||
if [[ "$DB_URL" == *"?"* ]]; then
|
||||
DB_URL="${DB_URL}&sslmode=disable"
|
||||
else
|
||||
DB_URL="${DB_URL}?sslmode=disable"
|
||||
fi
|
||||
elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then
|
||||
# Replace sslmode=require/prefer with sslmode=disable for local dev
|
||||
DB_URL="${DB_URL//sslmode=require/sslmode=disable}"
|
||||
DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}"
|
||||
fi
|
||||
else
|
||||
# Use Python to properly URL encode password if it contains special characters
|
||||
# This handles special characters in passwords correctly
|
||||
if command -v python3 &> /dev/null; then
|
||||
ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))")
|
||||
else
|
||||
# Fallback: try to use printf for basic encoding (may not work for all special chars)
|
||||
ENCODED_PASSWORD="$DB_PASSWORD"
|
||||
fi
|
||||
DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
|
||||
fi
|
||||
|
||||
# Execute migration based on command
|
||||
case "$1" in
|
||||
|
||||
Reference in New Issue
Block a user