feat: Integrate LLM-based reranking in KnowledgeSearchTool

- Added optional chat model to KnowledgeSearchTool for enhanced reranking using LLM prompt scoring.
- Updated rerankResults method to prioritize LLM-based reranking over traditional rerank model.
- Implemented new functions for LLM scoring and parsing scores from responses, improving search result relevance.
- Enhanced migration script to handle database versioning and dirty state checks more robustly.
This commit is contained in:
wizardchen
2025-11-13 01:22:08 +08:00
parent 7be09acb9e
commit 9f6b903ef2
5 changed files with 303 additions and 11 deletions
+200 -3
View File
@@ -5,9 +5,12 @@ import (
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -27,6 +30,7 @@ type KnowledgeSearchTool struct {
tenantID uint
allowedKBs []string
rerankModel rerank.Reranker
chatModel chat.Chat // Optional chat model for LLM-based reranking
}
// NewKnowledgeSearchTool creates a new knowledge search tool
@@ -35,6 +39,7 @@ func NewKnowledgeSearchTool(
tenantID uint,
allowedKBs []string,
rerankModel rerank.Reranker,
chatModel chat.Chat,
) *KnowledgeSearchTool {
description := `Search within knowledge bases with flexible query modes. Unified tool that supports both targeted and broad searches.
@@ -108,6 +113,7 @@ func NewKnowledgeSearchTool(
tenantID: tenantID,
allowedKBs: allowedKBs,
rerankModel: rerankModel,
chatModel: chatModel,
}
}
@@ -346,7 +352,29 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
len(filteredResults), len(allResults))
// Apply ReRank if model is configured
if t.rerankModel != nil && len(filteredResults) > 0 {
// Prefer chatModel (LLM-based reranking) over rerankModel if both are available
if t.chatModel != nil && len(filteredResults) > 0 {
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying LLM-based rerank with model: %s, input: %d results",
t.chatModel.GetModelName(), len(filteredResults))
rerankQuery := singleQuery
if rerankQuery == "" && len(vectorQueries) > 0 {
rerankQuery = vectorQueries[0] // Use first vector query as rerank query
} else if rerankQuery == "" && len(keywordQueries) > 0 {
rerankQuery = keywordQueries[0] // Use first keyword query as fallback
}
if rerankQuery != "" {
logger.Debugf(ctx, "[Tool][KnowledgeSearch] Rerank query: %s", rerankQuery)
rerankedResults, err := t.rerankResults(ctx, rerankQuery, filteredResults)
if err != nil {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] LLM rerank failed, using original results: %v", err)
} else {
filteredResults = rerankedResults
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM rerank completed successfully: %d results",
len(filteredResults))
}
}
} else if t.rerankModel != nil && len(filteredResults) > 0 {
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank with model: %s, input: %d results",
t.rerankModel.GetModelName(), len(filteredResults))
rerankQuery := singleQuery
@@ -560,11 +588,180 @@ func (t *KnowledgeSearchTool) filterByThreshold(
return filtered
}
// rerankResults applies reranking to search results using the configured rerank model
// rerankResults applies reranking to search results using LLM prompt scoring or rerank model
func (t *KnowledgeSearchTool) rerankResults(
ctx context.Context,
query string,
results []*searchResultWithMeta,
) ([]*searchResultWithMeta, error) {
// If chatModel is available, use LLM-based prompt scoring
if t.chatModel != nil {
return t.rerankWithLLM(ctx, query, results)
}
// Fallback to rerank model if available
if t.rerankModel != nil {
return t.rerankWithModel(ctx, query, results)
}
// No reranking available, return original results
return results, nil
}
// rerankWithLLM uses LLM prompt to score and rerank search results
func (t *KnowledgeSearchTool) rerankWithLLM(
ctx context.Context,
query string,
results []*searchResultWithMeta,
) ([]*searchResultWithMeta, error) {
logger.Infof(ctx, "[Tool][KnowledgeSearch] Using LLM for reranking %d results", len(results))
// Build prompt with query and all passages
var passagesBuilder strings.Builder
for i, result := range results {
passagesBuilder.WriteString(fmt.Sprintf("Passage %d:\n%s\n\n", i+1, result.Content))
}
prompt := fmt.Sprintf(`你是一个相关性评分专家。请根据查询问题,对以下每个段落进行相关性评分。
查询问题:%s
评分要求:
1. 评分范围:0.0 到 1.0 之间的小数
2. 1.0 表示完全相关,0.0 表示完全不相关
3. 考虑语义相关性、信息完整性、准确性等因素
4. 必须为每个段落返回一个评分
5. 输出格式:每行一个评分,格式为 "Passage N: X.XX"(N 是段落编号,X.XX 是 0.0-1.0 之间的分数)
段落列表:
%s
请严格按照以下格式返回每个段落的评分(每行一个,共 %d 个):
Passage 1: X.XX
Passage 2: X.XX
...
Passage %d: X.XX`, query, passagesBuilder.String(), len(results), len(results))
messages := []chat.Message{
{
Role: "system",
Content: "你是一个专业的文本相关性评分专家,能够准确评估文本与查询问题的相关性。",
},
{
Role: "user",
Content: prompt,
},
}
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
Temperature: 0.1, // Low temperature for consistent scoring
MaxTokens: 1024,
})
if err != nil {
return nil, fmt.Errorf("LLM rerank call failed: %w", err)
}
// Parse scores from response
scores, err := t.parseScoresFromResponse(response.Content, len(results))
if err != nil {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to parse LLM scores: %v, using original scores", err)
return results, nil // Return original results if parsing fails
}
// Create reranked results with new scores
reranked := make([]*searchResultWithMeta, 0, len(results))
for i, result := range results {
newResult := *result
if i < len(scores) {
newResult.Score = scores[i]
}
reranked = append(reranked, &newResult)
}
// Sort by new scores (descending)
sort.Slice(reranked, func(i, j int) bool {
return reranked[i].Score > reranked[j].Score
})
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d results from %d original results", len(reranked), len(results))
return reranked, nil
}
// parseScoresFromResponse parses scores from LLM response text
func (t *KnowledgeSearchTool) parseScoresFromResponse(responseText string, expectedCount int) ([]float64, error) {
lines := strings.Split(strings.TrimSpace(responseText), "\n")
scores := make([]float64, 0, expectedCount)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Try to extract score from various formats:
// "Passage 1: 0.85"
// "1: 0.85"
// "0.85"
// etc.
parts := strings.Split(line, ":")
var scoreStr string
if len(parts) >= 2 {
scoreStr = strings.TrimSpace(parts[len(parts)-1])
} else {
scoreStr = strings.TrimSpace(line)
}
// Remove any non-numeric characters except decimal point
scoreStr = strings.TrimFunc(scoreStr, func(r rune) bool {
return (r < '0' || r > '9') && r != '.'
})
if scoreStr == "" {
continue
}
score, err := strconv.ParseFloat(scoreStr, 64)
if err != nil {
continue // Skip invalid scores
}
// Clamp score to [0.0, 1.0]
if score < 0.0 {
score = 0.0
}
if score > 1.0 {
score = 1.0
}
scores = append(scores, score)
}
if len(scores) == 0 {
return nil, fmt.Errorf("no valid scores found in response")
}
// If we got fewer scores than expected, pad with last score or 0.5
for len(scores) < expectedCount {
if len(scores) > 0 {
scores = append(scores, scores[len(scores)-1])
} else {
scores = append(scores, 0.5)
}
}
// Truncate if we got more scores than expected
if len(scores) > expectedCount {
scores = scores[:expectedCount]
}
return scores, nil
}
// rerankWithModel uses the rerank model for reranking (fallback)
func (t *KnowledgeSearchTool) rerankWithModel(
ctx context.Context,
query string,
results []*searchResultWithMeta,
) ([]*searchResultWithMeta, error) {
// Prepare passages for reranking
passages := make([]string, len(results))
@@ -589,7 +786,7 @@ func (t *KnowledgeSearchTool) rerankResults(
}
}
logger.Infof(ctx, "Reranked %d results from %d original results", len(reranked), len(results))
logger.Infof(ctx, "[Tool][KnowledgeSearch] Reranked %d results from %d original results", len(reranked), len(results))
return reranked, nil
}
@@ -208,7 +208,7 @@ func (s *agentService) registerTools(
case "todo_write":
registry.RegisterTool(tools.NewTodoWriteTool())
case "knowledge_search":
registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel))
registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel, chatModel))
case "get_related_chunks":
registry.RegisterTool(tools.NewGetRelatedChunksTool(s.chunkService, s.knowledgeBaseService))
case "query_knowledge_graph":
+50 -2
View File
@@ -23,11 +23,59 @@ func RunMigrations(dsn string) error {
}
defer m.Close()
// Get current version before migration
oldVersion, _, _ := m.Version()
// Check current version and dirty state before migration
oldVersion, oldDirty, versionErr := m.Version()
if versionErr != nil && versionErr != migrate.ErrNilVersion {
// If we can't get version and it's not because database is empty, return error
return fmt.Errorf("failed to get migration version: %w", versionErr)
}
// If database is in dirty state, return detailed error with fix instructions
if oldDirty {
// Calculate the version to force to (usually the previous version)
forceVersion := int(oldVersion) - 1
if oldVersion == 0 || forceVersion < 0 {
forceVersion = 0
}
return fmt.Errorf(
"database is in dirty state at version %d. This usually means a migration failed partway through. "+
"To fix this:\n"+
"1. Check if the migration partially applied changes and manually fix if needed\n"+
"2. Use the force command to set the version to the last successful migration (usually %d):\n"+
" ./scripts/migrate.sh force %d\n"+
" Or if using make: make migrate-force version=%d\n"+
"3. After fixing, restart the application to retry the migration",
oldVersion,
forceVersion,
forceVersion,
forceVersion,
)
}
// Run all pending migrations
if err := m.Up(); err != nil && err != migrate.ErrNoChange {
// Check if error is due to dirty state (in case it became dirty during migration)
currentVersion, currentDirty, versionCheckErr := m.Version()
if versionCheckErr == nil && currentDirty {
// Calculate the version to force to (usually the previous version)
forceVersion := currentVersion - 1
if currentVersion == 0 {
forceVersion = 0
}
return fmt.Errorf(
"migration failed and database is now in dirty state at version %d. "+
"To fix this:\n"+
"1. Check if the migration partially applied changes and manually fix if needed\n"+
"2. Use the force command to set the version to the last successful migration (usually %d):\n"+
" ./scripts/migrate.sh force %d\n"+
" Or if using make: make migrate-force version=%d\n"+
"3. After fixing, restart the application to retry the migration",
currentVersion,
forceVersion,
forceVersion,
forceVersion,
)
}
return fmt.Errorf("failed to run migrations: %w", err)
}
@@ -3,11 +3,20 @@
BEGIN;
-- Add web_search_config column to tenants table
ALTER TABLE tenants
ADD COLUMN web_search_config JSONB DEFAULT NULL;
-- Add web_search_config column to tenants table (if not exists)
-- Note: PostgreSQL 9.6+ supports IF NOT EXISTS
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'tenants' AND column_name = 'web_search_config'
) THEN
ALTER TABLE tenants
ADD COLUMN web_search_config JSONB DEFAULT NULL;
END IF;
END $$;
-- Add comment
-- Add or update comment
COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant';
COMMIT;
+39 -1
View File
@@ -1,6 +1,18 @@
#!/bin/bash
set -e
# Get the script directory and project root
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
# Load .env file if it exists (for development mode)
if [ -f "$PROJECT_ROOT/.env" ]; then
echo "Loading .env file from $PROJECT_ROOT/.env"
set -a
source "$PROJECT_ROOT/.env"
set +a
fi
# Database connection details (can be overridden by environment variables)
DB_HOST=${DB_HOST:-localhost}
DB_PORT=${DB_PORT:-5432}
@@ -19,7 +31,33 @@ if ! command -v migrate &> /dev/null; then
fi
# Construct the database URL
DB_URL="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
# If DB_URL is already set in .env, use it but ensure sslmode=disable is set
# Otherwise, construct it from individual components
if [ -n "$DB_URL" ]; then
# If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified)
if [[ "$DB_URL" != *"sslmode="* ]]; then
# Add sslmode=disable if not present
if [[ "$DB_URL" == *"?"* ]]; then
DB_URL="${DB_URL}&sslmode=disable"
else
DB_URL="${DB_URL}?sslmode=disable"
fi
elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then
# Replace sslmode=require/prefer with sslmode=disable for local dev
DB_URL="${DB_URL//sslmode=require/sslmode=disable}"
DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}"
fi
else
# Use Python to properly URL encode password if it contains special characters
# This handles special characters in passwords correctly
if command -v python3 &> /dev/null; then
ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))")
else
# Fallback: try to use printf for basic encoding (may not work for all special chars)
ENCODED_PASSWORD="$DB_PASSWORD"
fi
DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable"
fi
# Execute migration based on command
case "$1" in