fix(rerank): stop sending vLLM truncate_prompt_tokens by default
The generic OpenAI-compatible reranker hardcoded `truncate_prompt_tokens: 511` on every /rerank request. Providers that honor this vLLM-specific field (e.g. SiliconFlow serving Qwen3-Reranker) keep only the last 511 tokens of the templated rerank prompt, cutting the instruction and query off any document longer than ~511 tokens. Every relevance score then collapses to near zero (~0.0067 instead of ~0.998), falls below the score threshold, and knowledge-base search returns empty results. A direct curl without the field returns the correct scores. WeKnora already removed this exact hardcoded parameter from the embedding path in #960; the rerank path was never fixed. - Add `omitempty` to `truncate_prompt_tokens` and `additional_data` so empty optional fields are not sent. - Make truncation opt-in via `extra_config.truncate_prompt_tokens` (validated > 0 at construction), preserving an escape hatch for self-hosted vLLM deployments with small-context rerankers. - Add regression tests reproducing the collapsed-score symptom and the opt-in path. Fixes #2143
This commit is contained in:
committed by
lyingbug
parent
e289595054
commit
83da0903b6
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
@@ -20,6 +22,12 @@ type OpenAIReranker struct {
|
||||
baseURL string // Base URL for API requests
|
||||
client *http.Client // HTTP client for making API requests
|
||||
customHeaders map[string]string
|
||||
// truncatePromptTokens, when > 0, is sent as the vLLM-specific
|
||||
// truncate_prompt_tokens request field. It must never be sent by default:
|
||||
// providers that honor it (e.g. SiliconFlow) keep only the LAST N tokens of
|
||||
// the templated rerank prompt, which cuts the query off long documents and
|
||||
// collapses every relevance score to near zero (issue #2143).
|
||||
truncatePromptTokens int
|
||||
}
|
||||
|
||||
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||
@@ -29,11 +37,11 @@ func (r *OpenAIReranker) SetCustomHeaders(headers map[string]string) {
|
||||
|
||||
// RerankRequest represents a request to rerank documents based on relevance to a query
|
||||
type RerankRequest struct {
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
AdditionalData map[string]interface{} `json:"additional_data"` // Optional additional data for the model
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens"` // Maximum prompt tokens to use
|
||||
Model string `json:"model"` // Model to use for reranking
|
||||
Query string `json:"query"` // Query text to compare documents against
|
||||
Documents []string `json:"documents"` // List of document texts to rerank
|
||||
AdditionalData map[string]interface{} `json:"additional_data,omitempty"` // Optional additional data for the model
|
||||
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` // Maximum prompt tokens to use (vLLM-specific, opt-in)
|
||||
}
|
||||
|
||||
// RerankResponse represents the response from a reranking request
|
||||
@@ -60,23 +68,39 @@ func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Optional opt-in for vLLM-style deployments that need server-side prompt
|
||||
// truncation. Configured via extra_config; never enabled by default.
|
||||
truncatePromptTokens := 0
|
||||
if config.ExtraConfig != nil {
|
||||
if raw := strings.TrimSpace(config.ExtraConfig["truncate_prompt_tokens"]); raw != "" {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
return nil, fmt.Errorf("invalid truncate_prompt_tokens in extra_config: %q", raw)
|
||||
}
|
||||
truncatePromptTokens = n
|
||||
}
|
||||
}
|
||||
|
||||
return &OpenAIReranker{
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
modelName: config.ModelName,
|
||||
modelID: config.ModelID,
|
||||
apiKey: apiKey,
|
||||
baseURL: baseURL,
|
||||
client: newRerankHTTPClient(0),
|
||||
truncatePromptTokens: truncatePromptTokens,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Rerank performs document reranking based on relevance to the query
|
||||
func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
|
||||
// Build the request body
|
||||
// Build the request body. truncate_prompt_tokens is only included when
|
||||
// explicitly configured: sending it unconditionally corrupts scores on
|
||||
// providers that honor it (see OpenAIReranker.truncatePromptTokens).
|
||||
requestBody := &RerankRequest{
|
||||
Model: r.modelName,
|
||||
Query: query,
|
||||
Documents: documents,
|
||||
TruncatePromptTokens: 511,
|
||||
TruncatePromptTokens: r.truncatePromptTokens,
|
||||
}
|
||||
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
|
||||
Reference in New Issue
Block a user