From 9f6b903ef27bb533ac5611221cb257e0c22c8dee Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 13 Nov 2025 01:22:08 +0800 Subject: [PATCH] feat: Integrate LLM-based reranking in KnowledgeSearchTool - Added optional chat model to KnowledgeSearchTool for enhanced reranking using LLM prompt scoring. - Updated rerankResults method to prioritize LLM-based reranking over traditional rerank model. - Implemented new functions for LLM scoring and parsing scores from responses, improving search result relevance. - Enhanced migration script to handle database versioning and dirty state checks more robustly. --- internal/agent/tools/knowledge_search.go | 203 +++++++++++++++++- internal/application/service/agent_service.go | 2 +- internal/database/migration.go | 52 ++++- .../000005_add_web_search_config.up.sql | 17 +- scripts/migrate.sh | 40 +++- 5 files changed, 303 insertions(+), 11 deletions(-) diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index b80ec24a..9fba8b8a 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -5,9 +5,12 @@ import ( "encoding/json" "fmt" "sort" + "strconv" + "strings" "sync" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/rerank" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -27,6 +30,7 @@ type KnowledgeSearchTool struct { tenantID uint allowedKBs []string rerankModel rerank.Reranker + chatModel chat.Chat // Optional chat model for LLM-based reranking } // NewKnowledgeSearchTool creates a new knowledge search tool @@ -35,6 +39,7 @@ func NewKnowledgeSearchTool( tenantID uint, allowedKBs []string, rerankModel rerank.Reranker, + chatModel chat.Chat, ) *KnowledgeSearchTool { description := `Search within knowledge bases with flexible query modes. Unified tool that supports both targeted and broad searches. @@ -108,6 +113,7 @@ func NewKnowledgeSearchTool( tenantID: tenantID, allowedKBs: allowedKBs, rerankModel: rerankModel, + chatModel: chatModel, } } @@ -346,7 +352,29 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter len(filteredResults), len(allResults)) // Apply ReRank if model is configured - if t.rerankModel != nil && len(filteredResults) > 0 { + // Prefer chatModel (LLM-based reranking) over rerankModel if both are available + if t.chatModel != nil && len(filteredResults) > 0 { + logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying LLM-based rerank with model: %s, input: %d results", + t.chatModel.GetModelName(), len(filteredResults)) + rerankQuery := singleQuery + if rerankQuery == "" && len(vectorQueries) > 0 { + rerankQuery = vectorQueries[0] // Use first vector query as rerank query + } else if rerankQuery == "" && len(keywordQueries) > 0 { + rerankQuery = keywordQueries[0] // Use first keyword query as fallback + } + + if rerankQuery != "" { + logger.Debugf(ctx, "[Tool][KnowledgeSearch] Rerank query: %s", rerankQuery) + rerankedResults, err := t.rerankResults(ctx, rerankQuery, filteredResults) + if err != nil { + logger.Warnf(ctx, "[Tool][KnowledgeSearch] LLM rerank failed, using original results: %v", err) + } else { + filteredResults = rerankedResults + logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM rerank completed successfully: %d results", + len(filteredResults)) + } + } + } else if t.rerankModel != nil && len(filteredResults) > 0 { logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank with model: %s, input: %d results", t.rerankModel.GetModelName(), len(filteredResults)) rerankQuery := singleQuery @@ -560,11 +588,180 @@ func (t *KnowledgeSearchTool) filterByThreshold( return filtered } -// rerankResults applies reranking to search results using the configured rerank model +// rerankResults applies reranking to search results using LLM prompt scoring or rerank model func (t *KnowledgeSearchTool) rerankResults( ctx context.Context, query string, results []*searchResultWithMeta, +) ([]*searchResultWithMeta, error) { + // If chatModel is available, use LLM-based prompt scoring + if t.chatModel != nil { + return t.rerankWithLLM(ctx, query, results) + } + + // Fallback to rerank model if available + if t.rerankModel != nil { + return t.rerankWithModel(ctx, query, results) + } + + // No reranking available, return original results + return results, nil +} + +// rerankWithLLM uses LLM prompt to score and rerank search results +func (t *KnowledgeSearchTool) rerankWithLLM( + ctx context.Context, + query string, + results []*searchResultWithMeta, +) ([]*searchResultWithMeta, error) { + logger.Infof(ctx, "[Tool][KnowledgeSearch] Using LLM for reranking %d results", len(results)) + + // Build prompt with query and all passages + var passagesBuilder strings.Builder + for i, result := range results { + passagesBuilder.WriteString(fmt.Sprintf("Passage %d:\n%s\n\n", i+1, result.Content)) + } + + prompt := fmt.Sprintf(`你是一个相关性评分专家。请根据查询问题,对以下每个段落进行相关性评分。 + +查询问题:%s + +评分要求: +1. 评分范围:0.0 到 1.0 之间的小数 +2. 1.0 表示完全相关,0.0 表示完全不相关 +3. 考虑语义相关性、信息完整性、准确性等因素 +4. 必须为每个段落返回一个评分 +5. 输出格式:每行一个评分,格式为 "Passage N: X.XX"(N 是段落编号,X.XX 是 0.0-1.0 之间的分数) + +段落列表: +%s + +请严格按照以下格式返回每个段落的评分(每行一个,共 %d 个): +Passage 1: X.XX +Passage 2: X.XX +... +Passage %d: X.XX`, query, passagesBuilder.String(), len(results), len(results)) + + messages := []chat.Message{ + { + Role: "system", + Content: "你是一个专业的文本相关性评分专家,能够准确评估文本与查询问题的相关性。", + }, + { + Role: "user", + Content: prompt, + }, + } + + response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{ + Temperature: 0.1, // Low temperature for consistent scoring + MaxTokens: 1024, + }) + if err != nil { + return nil, fmt.Errorf("LLM rerank call failed: %w", err) + } + + // Parse scores from response + scores, err := t.parseScoresFromResponse(response.Content, len(results)) + if err != nil { + logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to parse LLM scores: %v, using original scores", err) + return results, nil // Return original results if parsing fails + } + + // Create reranked results with new scores + reranked := make([]*searchResultWithMeta, 0, len(results)) + for i, result := range results { + newResult := *result + if i < len(scores) { + newResult.Score = scores[i] + } + reranked = append(reranked, &newResult) + } + + // Sort by new scores (descending) + sort.Slice(reranked, func(i, j int) bool { + return reranked[i].Score > reranked[j].Score + }) + + logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d results from %d original results", len(reranked), len(results)) + return reranked, nil +} + +// parseScoresFromResponse parses scores from LLM response text +func (t *KnowledgeSearchTool) parseScoresFromResponse(responseText string, expectedCount int) ([]float64, error) { + lines := strings.Split(strings.TrimSpace(responseText), "\n") + scores := make([]float64, 0, expectedCount) + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // Try to extract score from various formats: + // "Passage 1: 0.85" + // "1: 0.85" + // "0.85" + // etc. + parts := strings.Split(line, ":") + var scoreStr string + if len(parts) >= 2 { + scoreStr = strings.TrimSpace(parts[len(parts)-1]) + } else { + scoreStr = strings.TrimSpace(line) + } + + // Remove any non-numeric characters except decimal point + scoreStr = strings.TrimFunc(scoreStr, func(r rune) bool { + return (r < '0' || r > '9') && r != '.' + }) + + if scoreStr == "" { + continue + } + + score, err := strconv.ParseFloat(scoreStr, 64) + if err != nil { + continue // Skip invalid scores + } + + // Clamp score to [0.0, 1.0] + if score < 0.0 { + score = 0.0 + } + if score > 1.0 { + score = 1.0 + } + + scores = append(scores, score) + } + + if len(scores) == 0 { + return nil, fmt.Errorf("no valid scores found in response") + } + + // If we got fewer scores than expected, pad with last score or 0.5 + for len(scores) < expectedCount { + if len(scores) > 0 { + scores = append(scores, scores[len(scores)-1]) + } else { + scores = append(scores, 0.5) + } + } + + // Truncate if we got more scores than expected + if len(scores) > expectedCount { + scores = scores[:expectedCount] + } + + return scores, nil +} + +// rerankWithModel uses the rerank model for reranking (fallback) +func (t *KnowledgeSearchTool) rerankWithModel( + ctx context.Context, + query string, + results []*searchResultWithMeta, ) ([]*searchResultWithMeta, error) { // Prepare passages for reranking passages := make([]string, len(results)) @@ -589,7 +786,7 @@ func (t *KnowledgeSearchTool) rerankResults( } } - logger.Infof(ctx, "Reranked %d results from %d original results", len(reranked), len(results)) + logger.Infof(ctx, "[Tool][KnowledgeSearch] Reranked %d results from %d original results", len(reranked), len(results)) return reranked, nil } diff --git a/internal/application/service/agent_service.go b/internal/application/service/agent_service.go index 0aad36f0..66008872 100644 --- a/internal/application/service/agent_service.go +++ b/internal/application/service/agent_service.go @@ -208,7 +208,7 @@ func (s *agentService) registerTools( case "todo_write": registry.RegisterTool(tools.NewTodoWriteTool()) case "knowledge_search": - registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel)) + registry.RegisterTool(tools.NewKnowledgeSearchTool(s.knowledgeBaseService, tenantID, config.KnowledgeBases, rerankModel, chatModel)) case "get_related_chunks": registry.RegisterTool(tools.NewGetRelatedChunksTool(s.chunkService, s.knowledgeBaseService)) case "query_knowledge_graph": diff --git a/internal/database/migration.go b/internal/database/migration.go index 3f2e1e53..47ba3b9a 100644 --- a/internal/database/migration.go +++ b/internal/database/migration.go @@ -23,11 +23,59 @@ func RunMigrations(dsn string) error { } defer m.Close() - // Get current version before migration - oldVersion, _, _ := m.Version() + // Check current version and dirty state before migration + oldVersion, oldDirty, versionErr := m.Version() + if versionErr != nil && versionErr != migrate.ErrNilVersion { + // If we can't get version and it's not because database is empty, return error + return fmt.Errorf("failed to get migration version: %w", versionErr) + } + + // If database is in dirty state, return detailed error with fix instructions + if oldDirty { + // Calculate the version to force to (usually the previous version) + forceVersion := int(oldVersion) - 1 + if oldVersion == 0 || forceVersion < 0 { + forceVersion = 0 + } + return fmt.Errorf( + "database is in dirty state at version %d. This usually means a migration failed partway through. "+ + "To fix this:\n"+ + "1. Check if the migration partially applied changes and manually fix if needed\n"+ + "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ + " ./scripts/migrate.sh force %d\n"+ + " Or if using make: make migrate-force version=%d\n"+ + "3. After fixing, restart the application to retry the migration", + oldVersion, + forceVersion, + forceVersion, + forceVersion, + ) + } // Run all pending migrations if err := m.Up(); err != nil && err != migrate.ErrNoChange { + // Check if error is due to dirty state (in case it became dirty during migration) + currentVersion, currentDirty, versionCheckErr := m.Version() + if versionCheckErr == nil && currentDirty { + // Calculate the version to force to (usually the previous version) + forceVersion := currentVersion - 1 + if currentVersion == 0 { + forceVersion = 0 + } + return fmt.Errorf( + "migration failed and database is now in dirty state at version %d. "+ + "To fix this:\n"+ + "1. Check if the migration partially applied changes and manually fix if needed\n"+ + "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ + " ./scripts/migrate.sh force %d\n"+ + " Or if using make: make migrate-force version=%d\n"+ + "3. After fixing, restart the application to retry the migration", + currentVersion, + forceVersion, + forceVersion, + forceVersion, + ) + } return fmt.Errorf("failed to run migrations: %w", err) } diff --git a/migrations/versioned/000005_add_web_search_config.up.sql b/migrations/versioned/000005_add_web_search_config.up.sql index 0a2ac788..3f4f92f0 100644 --- a/migrations/versioned/000005_add_web_search_config.up.sql +++ b/migrations/versioned/000005_add_web_search_config.up.sql @@ -3,11 +3,20 @@ BEGIN; --- Add web_search_config column to tenants table -ALTER TABLE tenants -ADD COLUMN web_search_config JSONB DEFAULT NULL; +-- Add web_search_config column to tenants table (if not exists) +-- Note: PostgreSQL 9.6+ supports IF NOT EXISTS +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'tenants' AND column_name = 'web_search_config' + ) THEN + ALTER TABLE tenants + ADD COLUMN web_search_config JSONB DEFAULT NULL; + END IF; +END $$; --- Add comment +-- Add or update comment COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant'; COMMIT; diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 9165e6d1..e4b182ab 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -1,6 +1,18 @@ #!/bin/bash set -e +# Get the script directory and project root +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" + +# Load .env file if it exists (for development mode) +if [ -f "$PROJECT_ROOT/.env" ]; then + echo "Loading .env file from $PROJECT_ROOT/.env" + set -a + source "$PROJECT_ROOT/.env" + set +a +fi + # Database connection details (can be overridden by environment variables) DB_HOST=${DB_HOST:-localhost} DB_PORT=${DB_PORT:-5432} @@ -19,7 +31,33 @@ if ! command -v migrate &> /dev/null; then fi # Construct the database URL -DB_URL="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" +# If DB_URL is already set in .env, use it but ensure sslmode=disable is set +# Otherwise, construct it from individual components +if [ -n "$DB_URL" ]; then + # If DB_URL already exists, ensure sslmode=disable is set (unless sslmode is already specified) + if [[ "$DB_URL" != *"sslmode="* ]]; then + # Add sslmode=disable if not present + if [[ "$DB_URL" == *"?"* ]]; then + DB_URL="${DB_URL}&sslmode=disable" + else + DB_URL="${DB_URL}?sslmode=disable" + fi + elif [[ "$DB_URL" == *"sslmode=require"* ]] || [[ "$DB_URL" == *"sslmode=prefer"* ]]; then + # Replace sslmode=require/prefer with sslmode=disable for local dev + DB_URL="${DB_URL//sslmode=require/sslmode=disable}" + DB_URL="${DB_URL//sslmode=prefer/sslmode=disable}" + fi +else + # Use Python to properly URL encode password if it contains special characters + # This handles special characters in passwords correctly + if command -v python3 &> /dev/null; then + ENCODED_PASSWORD=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$DB_PASSWORD', safe=''))") + else + # Fallback: try to use printf for basic encoding (may not work for all special chars) + ENCODED_PASSWORD="$DB_PASSWORD" + fi + DB_URL="postgres://${DB_USER}:${ENCODED_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" +fi # Execute migration based on command case "$1" in