fix(doris): inline LIMIT/OFFSET as literals and enable parameter interpolation
Problem: Doris does not support ? placeholders for LIMIT and OFFSET clauses, causing VectorRetrieve, KeywordsRetrieve, and CopyIndices queries to fail at runtime with SQL parse errors. Root cause: 1. Query builders passed TopK/pageSize/offset as ? bind parameters (valid in MySQL/PostgreSQL but not in Doris's SQL dialect) 2. Doris MySQL driver was not configured to interpolate parameters, preventing automatic client-side placeholder replacement Fix: 1. Enable parameter interpolation in Doris connection DSN: - Add &interpolateParams=true to mysql.Open() DSN in container.go - This enables driver-level parameter substitution at query time 2. Inline LIMIT/OFFSET as string literals in Doris queries: - VectorRetrieve: LIMIT ? → LIMIT %d (params.TopK) - KeywordsRetrieve: LIMIT ? → LIMIT %d (params.TopK) - CopyIndices: LIMIT ? OFFSET ? → LIMIT %d OFFSET %d (pageSize, offset) - Remove inlined values from args slice Result: Queries are now built with literal LIMIT/OFFSET values instead of placeholders, compatible with Doris's SQL parser.
This commit is contained in:
@@ -327,7 +327,7 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
wb := buildBaseFilter(params)
|
||||
whereClause, whereArgs := wb.build()
|
||||
|
||||
// embedding 必须用字面量,threshold/topK 用占位符。
|
||||
// embedding 必须用字面量,Doris 不支持 LIMIT/OFFSET 使用占位符,必须内联为字面量。
|
||||
// 使用 HAVING 是因为 score 是 SELECT 列别名,WHERE 阶段还看不到。
|
||||
scoreExpr := fmt.Sprintf("inner_product_approximate(`%s`, %s)", fieldEmbedding, embeddingLiteral(queryEmbedding))
|
||||
if compatMode == dorisCompatModeLegacy {
|
||||
@@ -337,13 +337,14 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
"SELECT %s, %s AS score "+
|
||||
"FROM `%s` WHERE %s "+
|
||||
"HAVING score >= ? "+
|
||||
"ORDER BY score DESC LIMIT ?",
|
||||
"ORDER BY score DESC LIMIT %d",
|
||||
strings.Join(columnsForRetrieve, ", "),
|
||||
scoreExpr,
|
||||
table,
|
||||
whereClause,
|
||||
params.TopK,
|
||||
)
|
||||
args := append(whereArgs, params.Threshold, params.TopK)
|
||||
args := append(whereArgs, params.Threshold)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
@@ -386,11 +387,12 @@ func (r *dorisRepository) KeywordsRetrieve(ctx context.Context,
|
||||
var all []*types.IndexWithScore
|
||||
for _, table := range tables {
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s FROM `%s` WHERE %s AND %s MATCH_ANY ? LIMIT ?",
|
||||
"SELECT %s FROM `%s` WHERE %s AND %s MATCH_ANY ? LIMIT %d",
|
||||
strings.Join(columnsForRetrieve, ", "),
|
||||
table, whereClause, fieldContent,
|
||||
params.TopK,
|
||||
)
|
||||
args := append(append([]any{}, whereArgs...), query, params.TopK)
|
||||
args := append(append([]any{}, whereArgs...), query)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
@@ -442,11 +444,12 @@ func (r *dorisRepository) CopyIndices(ctx context.Context,
|
||||
|
||||
for {
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s FROM `%s` WHERE %s = ? ORDER BY %s LIMIT ? OFFSET ?",
|
||||
"SELECT %s FROM `%s` WHERE %s = ? ORDER BY %s LIMIT %d OFFSET %d",
|
||||
strings.Join(columnsForCopy, ", "),
|
||||
table, fieldKnowledgeBaseID, fieldID,
|
||||
pageSize, offset,
|
||||
)
|
||||
rows, err := r.db.QueryContext(ctx, stmt, sourceKnowledgeBaseID, pageSize, offset)
|
||||
rows, err := r.db.QueryContext(ctx, stmt, sourceKnowledgeBaseID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy indices scan: %w", err)
|
||||
}
|
||||
|
||||
@@ -1035,7 +1035,7 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local&interpolateParams=true",
|
||||
dorisUsername, dorisPassword, dorisAddr, dorisDatabase)
|
||||
dorisDB, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user