fix(rerank): stop sending vLLM truncate_prompt_tokens by default

The generic OpenAI-compatible reranker hardcoded `truncate_prompt_tokens:
511` on every /rerank request. Providers that honor this vLLM-specific
field (e.g. SiliconFlow serving Qwen3-Reranker) keep only the last 511
tokens of the templated rerank prompt, cutting the instruction and query
off any document longer than ~511 tokens. Every relevance score then
collapses to near zero (~0.0067 instead of ~0.998), falls below the score
threshold, and knowledge-base search returns empty results. A direct curl
without the field returns the correct scores.

WeKnora already removed this exact hardcoded parameter from the embedding
path in #960; the rerank path was never fixed.

- Add `omitempty` to `truncate_prompt_tokens` and `additional_data` so
  empty optional fields are not sent.
- Make truncation opt-in via `extra_config.truncate_prompt_tokens`
  (validated > 0 at construction), preserving an escape hatch for
  self-hosted vLLM deployments with small-context rerankers.
- Add regression tests reproducing the collapsed-score symptom and the
  opt-in path.

Fixes #2143
This commit is contained in:
Martín Alcalá Rubí
2026-07-20 23:38:27 -03:00
committed by lyingbug
parent e289595054
commit 83da0903b6
2 changed files with 193 additions and 12 deletions
+36 -12
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
secutils "github.com/Tencent/WeKnora/internal/utils"
@@ -20,6 +22,12 @@ type OpenAIReranker struct {
baseURL string // Base URL for API requests
client *http.Client // HTTP client for making API requests
customHeaders map[string]string
// truncatePromptTokens, when > 0, is sent as the vLLM-specific
// truncate_prompt_tokens request field. It must never be sent by default:
// providers that honor it (e.g. SiliconFlow) keep only the LAST N tokens of
// the templated rerank prompt, which cuts the query off long documents and
// collapses every relevance score to near zero (issue #2143).
truncatePromptTokens int
}
// SetCustomHeaders 设置用户自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
@@ -29,11 +37,11 @@ func (r *OpenAIReranker) SetCustomHeaders(headers map[string]string) {
// RerankRequest represents a request to rerank documents based on relevance to a query
type RerankRequest struct {
Model string `json:"model"` // Model to use for reranking
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
AdditionalData map[string]interface{} `json:"additional_data"` // Optional additional data for the model
TruncatePromptTokens int `json:"truncate_prompt_tokens"` // Maximum prompt tokens to use
Model string `json:"model"` // Model to use for reranking
Query string `json:"query"` // Query text to compare documents against
Documents []string `json:"documents"` // List of document texts to rerank
AdditionalData map[string]interface{} `json:"additional_data,omitempty"` // Optional additional data for the model
TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` // Maximum prompt tokens to use (vLLM-specific, opt-in)
}
// RerankResponse represents the response from a reranking request
@@ -60,23 +68,39 @@ func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) {
return nil, err
}
// Optional opt-in for vLLM-style deployments that need server-side prompt
// truncation. Configured via extra_config; never enabled by default.
truncatePromptTokens := 0
if config.ExtraConfig != nil {
if raw := strings.TrimSpace(config.ExtraConfig["truncate_prompt_tokens"]); raw != "" {
n, err := strconv.Atoi(raw)
if err != nil || n <= 0 {
return nil, fmt.Errorf("invalid truncate_prompt_tokens in extra_config: %q", raw)
}
truncatePromptTokens = n
}
}
return &OpenAIReranker{
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
modelName: config.ModelName,
modelID: config.ModelID,
apiKey: apiKey,
baseURL: baseURL,
client: newRerankHTTPClient(0),
truncatePromptTokens: truncatePromptTokens,
}, nil
}
// Rerank performs document reranking based on relevance to the query
func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) {
// Build the request body
// Build the request body. truncate_prompt_tokens is only included when
// explicitly configured: sending it unconditionally corrupts scores on
// providers that honor it (see OpenAIReranker.truncatePromptTokens).
requestBody := &RerankRequest{
Model: r.modelName,
Query: query,
Documents: documents,
TruncatePromptTokens: 511,
TruncatePromptTokens: r.truncatePromptTokens,
}
jsonData, err := json.Marshal(requestBody)