doris: add configurable compatibility modes and guard mode switches
Problem:
The hard-coded Doris vector function implementation (cosine_distance_approximate with
UNIQUE KEY ANN tables) fails on SelectDB 4.0.2-rc01 and other Doris builds lacking that
specific function support. Users had no way to adapt without code changes.
Root cause:
There is assumption all Doris deployments support the same vector function API, but different
builds (Doris OSS, SelectDB, Doris Cloud) ship with different function variants and table
key constraints. No capability detection or user configuration existed.
Solution:
Implement DORIS_COMPAT_MODE environment variable with three modes:
* auto (default/recommended): probe Doris server on first use to detect available vector
functions; prefer inner_product_duplicate (modern Doris 4.0+), fall back to legacy
(older builds lacking inner_product_approximate)
* legacy: hard-set to cosine_distance_approximate + UNIQUE KEY (for older Doris/SelectDB
builds without inner_product_approximate support)
* inner_product_duplicate: hard-set to inner_product_approximate + DUPLICATE KEY
(for modern Doris 4.0+ and current SelectDB with normalized embeddings)
Implementation details:
- add compat.go with one-time mode resolution (sync.Once) and capability probing
- inspect existing weknora_embeddings_* table DDL via SHOW CREATE TABLE to detect and
enforce schema compatibility; prevents silent mismatches
- fail fast with clear error message when configured mode does not match existing tables,
with explicit remediation steps (recreate tables or change env var)
- branch all query paths (inner_product_approximate vs cosine_distance_approximate),
DDL generation (DUPLICATE KEY vs UNIQUE KEY), write paths (embed normalization),
and chunk updates (Stream Load vs read-modify-write) by resolved compat mode
- add comprehensive repository tests for mode selection, auto-detection, and mismatch
scenarios; all tests pass
- expose DORIS_COMPAT_MODE in docker-compose.yml with auto as default
- document in .env.example with clear mode decision guidance
- log all mode decisions (requested, detected, probed, final) at INFO/WARN level
Key guarantee:
⚠️ DORIS_COMPAT_MODE is NOT interchangeable after embedding tables are created.
App will reject mode switches that conflict with existing table layout, preventing
silent data mismatches and query failures.
This commit is contained in:
@@ -545,6 +545,9 @@ DOCREADER_TRANSPORT=grpc
|
||||
# Doris 表前缀,对应每维度的物理表 <prefix>_<dim>
|
||||
# DORIS_TABLE_PREFIX=weknora_embeddings
|
||||
|
||||
# Doris compatibility mode: auto (probe server), legacy (cosine_distance), or inner_product_duplicate (inner_product); NOT interchangeable after tables are created.
|
||||
# DORIS_COMPAT_MODE=
|
||||
|
||||
# Tavily Search API Key(可选,启用 Tavily 网页搜索提供者)
|
||||
# TAVILY_API_KEY=tvly-your_tavily_api_key
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ services:
|
||||
- DORIS_USERNAME=${DORIS_USERNAME:-root}
|
||||
- DORIS_PASSWORD=${DORIS_PASSWORD:-}
|
||||
- DORIS_TABLE_PREFIX=${DORIS_TABLE_PREFIX:-weknora_embeddings}
|
||||
- DORIS_COMPAT_MODE=${DORIS_COMPAT_MODE:-auto}
|
||||
- STORAGE_TYPE=${STORAGE_TYPE:-}
|
||||
- LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-}
|
||||
- AUTO_RECOVER_DIRTY=${AUTO_RECOVER_DIRTY:-true}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
const envDorisCompatMode = "DORIS_COMPAT_MODE"
|
||||
|
||||
type dorisCompatMode string
|
||||
|
||||
const (
|
||||
dorisCompatModeAuto dorisCompatMode = "auto"
|
||||
dorisCompatModeLegacy dorisCompatMode = "legacy"
|
||||
dorisCompatModeInnerProductDuplicate dorisCompatMode = "inner_product_duplicate"
|
||||
)
|
||||
|
||||
type dorisCompatProbe struct {
|
||||
innerProductApproximate bool
|
||||
cosineDistanceApproximate bool
|
||||
}
|
||||
|
||||
func resolveConfiguredDorisCompatMode() (mode dorisCompatMode, invalid string) {
|
||||
raw := strings.TrimSpace(os.Getenv(envDorisCompatMode))
|
||||
if raw == "" {
|
||||
return dorisCompatModeAuto, ""
|
||||
}
|
||||
|
||||
switch strings.ToLower(raw) {
|
||||
case string(dorisCompatModeAuto):
|
||||
return dorisCompatModeAuto, ""
|
||||
case string(dorisCompatModeLegacy):
|
||||
return dorisCompatModeLegacy, ""
|
||||
case string(dorisCompatModeInnerProductDuplicate), "inner-product-duplicate", "inner_product", "inner-product":
|
||||
return dorisCompatModeInnerProductDuplicate, ""
|
||||
default:
|
||||
return dorisCompatModeAuto, raw
|
||||
}
|
||||
}
|
||||
|
||||
func (r *dorisRepository) resolveCompatMode(ctx context.Context) (dorisCompatMode, error) {
|
||||
r.compatResolveOnce.Do(func() {
|
||||
log := logger.GetLogger(ctx)
|
||||
requested := r.compatModeRequested
|
||||
if requested == "" {
|
||||
r.compatModeResolved = dorisCompatModeInnerProductDuplicate
|
||||
return
|
||||
}
|
||||
|
||||
existingMode, exampleTable, foundExisting, err := r.detectExistingCompatMode(ctx)
|
||||
if err != nil {
|
||||
r.compatResolveErr = err
|
||||
return
|
||||
}
|
||||
|
||||
if foundExisting {
|
||||
if requested != dorisCompatModeAuto && requested != existingMode {
|
||||
r.compatResolveErr = fmt.Errorf(
|
||||
"Doris compat mode %q does not match existing embedding tables (detected %q from %s). %s is not interchangeable after %s_* tables are created. Recreate the existing %s_* tables before switching modes, or set %s=%s",
|
||||
requested,
|
||||
existingMode,
|
||||
exampleTable,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
r.tableBaseName,
|
||||
envDorisCompatMode,
|
||||
existingMode,
|
||||
)
|
||||
log.Errorf("[Doris] %v", r.compatResolveErr)
|
||||
return
|
||||
}
|
||||
|
||||
r.compatModeResolved = existingMode
|
||||
log.Warnf(
|
||||
"[Doris] Using compat mode %s from existing embedding tables (%s). %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
||||
existingMode,
|
||||
exampleTable,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if requested == dorisCompatModeAuto {
|
||||
probe, probeErr := r.probeCompatMode(ctx)
|
||||
if probeErr != nil {
|
||||
r.compatResolveErr = probeErr
|
||||
log.Errorf("[Doris] %v", probeErr)
|
||||
return
|
||||
}
|
||||
log.Infof(
|
||||
"[Doris] Compat probe result: inner_product_approximate=%t, cosine_distance_approximate=%t",
|
||||
probe.innerProductApproximate,
|
||||
probe.cosineDistanceApproximate,
|
||||
)
|
||||
|
||||
switch {
|
||||
case probe.innerProductApproximate:
|
||||
r.compatModeResolved = dorisCompatModeInnerProductDuplicate
|
||||
case probe.cosineDistanceApproximate:
|
||||
r.compatModeResolved = dorisCompatModeLegacy
|
||||
default:
|
||||
r.compatResolveErr = fmt.Errorf(
|
||||
"Doris compatibility auto-detection could not find a supported vector function. Set %s=%s or %s=%s explicitly after verifying your Doris build. %s is not interchangeable after %s_* tables are created",
|
||||
envDorisCompatMode,
|
||||
dorisCompatModeInnerProductDuplicate,
|
||||
envDorisCompatMode,
|
||||
dorisCompatModeLegacy,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
log.Errorf("[Doris] %v", r.compatResolveErr)
|
||||
return
|
||||
}
|
||||
|
||||
log.Warnf(
|
||||
"[Doris] Auto-selected compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
||||
r.compatModeResolved,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
r.compatModeResolved = requested
|
||||
log.Warnf(
|
||||
"[Doris] Using configured compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
||||
r.compatModeResolved,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
})
|
||||
|
||||
return r.compatModeResolved, r.compatResolveErr
|
||||
}
|
||||
|
||||
func (r *dorisRepository) detectExistingCompatMode(ctx context.Context) (dorisCompatMode, string, bool, error) {
|
||||
tables, err := r.listEmbeddingTables(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("list Doris embedding tables: %w", err)
|
||||
}
|
||||
if len(tables) == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
sort.Strings(tables)
|
||||
|
||||
var detected dorisCompatMode
|
||||
for _, table := range tables {
|
||||
ddl, err := r.showCreateTable(ctx, table)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("show create table %s: %w", table, err)
|
||||
}
|
||||
mode, err := compatModeFromDDL(ddl)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("detect compat mode from %s: %w", table, err)
|
||||
}
|
||||
if detected == "" {
|
||||
detected = mode
|
||||
continue
|
||||
}
|
||||
if detected != mode {
|
||||
return "", "", false, fmt.Errorf(
|
||||
"existing Doris embedding tables use mixed compat modes (%s and %s). %s is not interchangeable after %s_* tables are created; recreate the existing %s_* tables with a single mode",
|
||||
detected,
|
||||
mode,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
r.tableBaseName,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return detected, tables[0], true, nil
|
||||
}
|
||||
|
||||
func (r *dorisRepository) showCreateTable(ctx context.Context, table string) (string, error) {
|
||||
row := r.db.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`", table))
|
||||
var tableName, ddl string
|
||||
if err := row.Scan(&tableName, &ddl); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ddl, nil
|
||||
}
|
||||
|
||||
func compatModeFromDDL(ddl string) (dorisCompatMode, error) {
|
||||
normalized := strings.ToLower(ddl)
|
||||
switch {
|
||||
case strings.Contains(normalized, "duplicate key("):
|
||||
return dorisCompatModeInnerProductDuplicate, nil
|
||||
case strings.Contains(normalized, "unique key("):
|
||||
return dorisCompatModeLegacy, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported table definition; expected UNIQUE KEY or DUPLICATE KEY")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *dorisRepository) probeCompatMode(ctx context.Context) (dorisCompatProbe, error) {
|
||||
probe := dorisCompatProbe{}
|
||||
probe.innerProductApproximate = r.vectorFunctionSupported(ctx, "inner_product_approximate([1.0],[1.0])")
|
||||
probe.cosineDistanceApproximate = r.vectorFunctionSupported(ctx, "cosine_distance_approximate([1.0],[1.0])")
|
||||
return probe, nil
|
||||
}
|
||||
|
||||
func (r *dorisRepository) vectorFunctionSupported(ctx context.Context, expr string) bool {
|
||||
var value sql.NullFloat64
|
||||
if err := r.db.QueryRowContext(ctx, "SELECT "+expr).Scan(&value); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (m dorisCompatMode) normalizeEmbeddings() bool {
|
||||
return m != dorisCompatModeLegacy
|
||||
}
|
||||
|
||||
func (m dorisCompatMode) usesReplaceWrite() bool {
|
||||
return m != dorisCompatModeLegacy
|
||||
}
|
||||
|
||||
func (m dorisCompatMode) usesRewriteChunkUpdates() bool {
|
||||
return m != dorisCompatModeLegacy
|
||||
}
|
||||
@@ -189,6 +189,28 @@ func validateEmbedding(vec []float32) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeEmbedding returns a unit-length copy of vec so Doris inner-product
|
||||
// ANN search can preserve cosine-style similarity semantics.
|
||||
func normalizeEmbedding(vec []float32) []float32 {
|
||||
if len(vec) == 0 {
|
||||
return nil
|
||||
}
|
||||
var sumSquares float64
|
||||
for _, value := range vec {
|
||||
f := float64(value)
|
||||
sumSquares += f * f
|
||||
}
|
||||
if sumSquares == 0 {
|
||||
return append([]float32(nil), vec...)
|
||||
}
|
||||
norm := float32(math.Sqrt(sumSquares))
|
||||
normalized := make([]float32, len(vec))
|
||||
for i, value := range vec {
|
||||
normalized[i] = value / norm
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
// errInvalidEmbedding 描述哪个下标含非有限值;用结构体而非 fmt.Errorf
|
||||
// 是为了让上层在日志里能拿到下标做问题定位。
|
||||
type errInvalidEmbedding struct {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -35,6 +36,10 @@ func NewDorisRetrieveEngineRepository(
|
||||
log.Info("[Doris] Initializing Doris retriever engine repository")
|
||||
|
||||
tableBaseName := types.ResolveCollectionName(indexCfg, envDorisTablePrefix, defaultTableBaseName)
|
||||
compatMode, invalidCompatMode := resolveConfiguredDorisCompatMode()
|
||||
if invalidCompatMode != "" {
|
||||
log.Warnf("[Doris] Invalid %s=%q, defaulting to %s", envDorisCompatMode, invalidCompatMode, dorisCompatModeAuto)
|
||||
}
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
@@ -46,9 +51,13 @@ func NewDorisRetrieveEngineRepository(
|
||||
tableBaseName: tableBaseName,
|
||||
bucketsNum: indexCfg.GetBucketsNum(0),
|
||||
replicationNum: indexCfg.GetReplicationNum(0),
|
||||
compatModeRequested: compatMode,
|
||||
}
|
||||
log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s, compat_mode=%s",
|
||||
database, tableBaseName, repo.feHTTPBase, repo.compatModeRequested)
|
||||
if os.Getenv(envDorisCompatMode) == "" {
|
||||
log.Infof("[Doris] %s not set, defaulting to %s and probing on first use", envDorisCompatMode, dorisCompatModeAuto)
|
||||
}
|
||||
log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s",
|
||||
database, tableBaseName, repo.feHTTPBase)
|
||||
return repo
|
||||
}
|
||||
|
||||
@@ -68,7 +77,7 @@ func (r *dorisRepository) EstimateStorageSize(_ context.Context,
|
||||
) int64 {
|
||||
var total int64
|
||||
for _, info := range indexInfoList {
|
||||
emb := toDorisVectorEmbedding(info, params)
|
||||
emb := toDorisVectorEmbedding(info, params, dorisCompatModeInnerProductDuplicate)
|
||||
total += calculateStorageSize(emb)
|
||||
}
|
||||
return total
|
||||
@@ -81,8 +90,8 @@ func (r *dorisRepository) Save(ctx context.Context,
|
||||
return r.BatchSave(ctx, []*types.IndexInfo{info}, additionalParams)
|
||||
}
|
||||
|
||||
// BatchSave 把同一批 IndexInfo 按维度分组,对每个维度构造一条
|
||||
// INSERT INTO ... VALUES (...), (...) 语句。UNIQUE KEY 表会自动按 id upsert。
|
||||
// BatchSave 把同一批 IndexInfo 按维度分组;Doris ANN 兼容表使用
|
||||
// DUPLICATE KEY,因此这里显式按 id 做 delete + insert,保持原先的替换语义。
|
||||
func (r *dorisRepository) BatchSave(ctx context.Context,
|
||||
indexInfoList []*types.IndexInfo, additionalParams map[string]any,
|
||||
) error {
|
||||
@@ -90,10 +99,14 @@ func (r *dorisRepository) BatchSave(ctx context.Context,
|
||||
if len(indexInfoList) == 0 {
|
||||
return nil
|
||||
}
|
||||
compatMode, err := r.resolveCompatMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
groups := make(map[int][]*DorisVectorEmbedding)
|
||||
for _, info := range indexInfoList {
|
||||
emb := toDorisVectorEmbedding(info, additionalParams)
|
||||
emb := toDorisVectorEmbedding(info, additionalParams, compatMode)
|
||||
if len(emb.Embedding) == 0 {
|
||||
log.Warnf("[Doris] Skipping empty embedding for chunk %s", info.ChunkID)
|
||||
continue
|
||||
@@ -117,7 +130,12 @@ func (r *dorisRepository) BatchSave(ctx context.Context,
|
||||
if err := r.ensureTable(ctx, dim); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.insertRows(ctx, r.getTableName(dim), rows); err != nil {
|
||||
if compatMode.usesReplaceWrite() {
|
||||
err = r.replaceRows(ctx, r.getTableName(dim), rows)
|
||||
} else {
|
||||
err = r.insertRows(ctx, r.getTableName(dim), rows)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch save dim=%d: %w", dim, err)
|
||||
}
|
||||
log.Infof("[Doris] Saved %d rows to %s", len(rows), r.getTableName(dim))
|
||||
@@ -157,6 +175,63 @@ func (r *dorisRepository) insertRows(ctx context.Context,
|
||||
return err
|
||||
}
|
||||
|
||||
// replaceRows 在 DUPLICATE KEY 表上显式模拟“按 id 覆盖”的语义。
|
||||
func (r *dorisRepository) replaceRows(ctx context.Context,
|
||||
table string, rows []*DorisVectorEmbedding,
|
||||
) error {
|
||||
rows = dedupeRowsByID(rows)
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
if err := r.deleteRowsByID(ctx, table, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.insertRows(ctx, table, rows)
|
||||
}
|
||||
|
||||
func (r *dorisRepository) deleteRowsByID(ctx context.Context,
|
||||
table string, ids []string,
|
||||
) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
placeholders := make([]string, len(ids))
|
||||
args := make([]any, len(ids))
|
||||
for i, id := range ids {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
stmt := fmt.Sprintf("DELETE FROM `%s` WHERE %s IN (%s)",
|
||||
table, fieldID, strings.Join(placeholders, ", "))
|
||||
_, err := r.db.ExecContext(ctx, stmt, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func dedupeRowsByID(rows []*DorisVectorEmbedding) []*DorisVectorEmbedding {
|
||||
if len(rows) < 2 {
|
||||
return rows
|
||||
}
|
||||
|
||||
out := make([]*DorisVectorEmbedding, 0, len(rows))
|
||||
positions := make(map[string]int, len(rows))
|
||||
for _, row := range rows {
|
||||
if idx, ok := positions[row.ID]; ok {
|
||||
out[idx] = row
|
||||
continue
|
||||
}
|
||||
positions[row.ID] = len(out)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList 用 chunk_id 列删除。dimension 用于定位具体表。
|
||||
func (r *dorisRepository) DeleteByChunkIDList(ctx context.Context,
|
||||
chunkIDList []string, dimension int, _ string,
|
||||
@@ -219,8 +294,9 @@ func (r *dorisRepository) Retrieve(ctx context.Context,
|
||||
return nil, fmt.Errorf("invalid retriever type: %v", params.RetrieverType)
|
||||
}
|
||||
|
||||
// VectorRetrieve 调用 cosine_distance_approximate 做 ANN 搜索,
|
||||
// score = 1 - distance 与 Qdrant cosine 相似度方向一致:值越大越相似。
|
||||
// VectorRetrieve 对查询向量先做单位化,再调用 inner_product_approximate 做
|
||||
// ANN 搜索;对单位向量而言,inner product 与 cosine similarity 等价,
|
||||
// 因此 score 仍然保持“越大越相似”的语义。
|
||||
func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
params types.RetrieveParams,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
@@ -228,6 +304,14 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
if err := validateEmbedding(params.Embedding); err != nil {
|
||||
return nil, fmt.Errorf("invalid query embedding: %w", err)
|
||||
}
|
||||
compatMode, err := r.resolveCompatMode(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queryEmbedding := append([]float32(nil), params.Embedding...)
|
||||
if compatMode.normalizeEmbeddings() {
|
||||
queryEmbedding = normalizeEmbedding(queryEmbedding)
|
||||
}
|
||||
dim := len(params.Embedding)
|
||||
table := r.getTableName(dim)
|
||||
|
||||
@@ -245,14 +329,17 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
|
||||
// embedding 必须用字面量,threshold/topK 用占位符。
|
||||
// 使用 HAVING 是因为 score 是 SELECT 列别名,WHERE 阶段还看不到。
|
||||
scoreExpr := fmt.Sprintf("inner_product_approximate(`%s`, %s)", fieldEmbedding, embeddingLiteral(queryEmbedding))
|
||||
if compatMode == dorisCompatModeLegacy {
|
||||
scoreExpr = fmt.Sprintf("(1 - cosine_distance_approximate(`%s`, %s))", fieldEmbedding, embeddingLiteral(queryEmbedding))
|
||||
}
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s, (1 - cosine_distance_approximate(`%s`, %s)) AS score "+
|
||||
"SELECT %s, %s AS score "+
|
||||
"FROM `%s` WHERE %s "+
|
||||
"HAVING score >= ? "+
|
||||
"ORDER BY score DESC LIMIT ?",
|
||||
strings.Join(columnsForRetrieve, ", "),
|
||||
fieldEmbedding,
|
||||
embeddingLiteral(params.Embedding),
|
||||
scoreExpr,
|
||||
table,
|
||||
whereClause,
|
||||
)
|
||||
@@ -260,7 +347,7 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("vector retrieve %s: %w", table, err)
|
||||
return nil, r.wrapVectorRetrieveError(table, compatMode, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
@@ -417,7 +504,7 @@ func (r *dorisRepository) CopyIndices(ctx context.Context,
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus / BatchUpdateChunkTagID 实际实现位于 streamload.go,
|
||||
// 通过 Stream Load partial update 协议执行高性能字段级更新。
|
||||
// 会按 compat mode 选择 partial update 或 rewrite rows。
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 私有辅助
|
||||
@@ -426,7 +513,12 @@ func (r *dorisRepository) CopyIndices(ctx context.Context,
|
||||
// toDorisVectorEmbedding 把 IndexInfo + 上层传入的 embedding 映射 转换为
|
||||
// Doris 行模型。Embedding 通过 additionalParams[fieldEmbedding] 中的
|
||||
// map[string][]float32 按 SourceID 取出,与 Qdrant/Milvus 完全一致。
|
||||
func toDorisVectorEmbedding(info *types.IndexInfo, additionalParams map[string]any) *DorisVectorEmbedding {
|
||||
// inner_product_duplicate 模式会先单位化,legacy 模式保留原始向量。
|
||||
func toDorisVectorEmbedding(
|
||||
info *types.IndexInfo,
|
||||
additionalParams map[string]any,
|
||||
compatMode dorisCompatMode,
|
||||
) *DorisVectorEmbedding {
|
||||
emb := &DorisVectorEmbedding{
|
||||
ID: info.ID,
|
||||
Content: info.Content,
|
||||
@@ -441,13 +533,32 @@ func toDorisVectorEmbedding(info *types.IndexInfo, additionalParams map[string]a
|
||||
if additionalParams != nil {
|
||||
if v, ok := additionalParams[fieldEmbedding]; ok {
|
||||
if m, ok := v.(map[string][]float32); ok {
|
||||
emb.Embedding = m[info.SourceID]
|
||||
emb.Embedding = append([]float32(nil), m[info.SourceID]...)
|
||||
if compatMode.normalizeEmbeddings() {
|
||||
emb.Embedding = normalizeEmbedding(emb.Embedding)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return emb
|
||||
}
|
||||
|
||||
func (r *dorisRepository) wrapVectorRetrieveError(table string, compatMode dorisCompatMode, err error) error {
|
||||
if compatMode == dorisCompatModeLegacy {
|
||||
return fmt.Errorf(
|
||||
"vector retrieve %s in Doris compat mode %s: %w. If your Doris build does not support cosine_distance_approximate or ANN on UNIQUE KEY tables, set %s=%s before creating embedding tables. %s is not interchangeable after %s_* tables are created",
|
||||
table,
|
||||
compatMode,
|
||||
err,
|
||||
envDorisCompatMode,
|
||||
dorisCompatModeInnerProductDuplicate,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
}
|
||||
return fmt.Errorf("vector retrieve %s in Doris compat mode %s: %w", table, compatMode, err)
|
||||
}
|
||||
|
||||
// translateSourceID 把源 SourceID 翻译到目标 SourceID,与 Qdrant 实现完全镜像:
|
||||
// - 普通 chunk:SourceID == ChunkID -> 使用 targetChunkID
|
||||
// - 生成型问题:SourceID == "<chunkID>-<questionID>" -> "<targetChunkID>-<questionID>"
|
||||
|
||||
@@ -58,6 +58,12 @@ func newTestRepo(t *testing.T) (*dorisRepository, sqlmock.Sqlmock, *httptest.Ser
|
||||
return repo, mock, srv, cleanup
|
||||
}
|
||||
|
||||
func primeCompatMode(repo *dorisRepository, mode dorisCompatMode, err error) {
|
||||
repo.compatModeResolved = mode
|
||||
repo.compatResolveErr = err
|
||||
repo.compatResolveOnce.Do(func() {})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// query.go:whereBuilder / embeddingLiteral / parseEmbeddingLiteral
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -259,7 +265,7 @@ func TestVectorRetrieve_SQLShape(t *testing.T) {
|
||||
WithArgs("weknora", "weknora_embeddings_3").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1))
|
||||
|
||||
mock.ExpectQuery(`SELECT id, content, .*cosine_distance_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`).
|
||||
mock.ExpectQuery(`SELECT id, content, .*inner_product_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`).
|
||||
WithArgs(true, 0.5, 5).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{
|
||||
@@ -283,6 +289,53 @@ func TestVectorRetrieve_SQLShape(t *testing.T) {
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestVectorRetrieve_SQLShape_LegacyMode(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
primeCompatMode(repo, dorisCompatModeLegacy, nil)
|
||||
|
||||
mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings_3").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1))
|
||||
|
||||
mock.ExpectQuery(`SELECT id, content, .*cosine_distance_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`).
|
||||
WithArgs(true, 0.5, 5).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{
|
||||
"id", "content", "source_id", "source_type",
|
||||
"chunk_id", "knowledge_id", "knowledge_base_id", "tag_id",
|
||||
"is_enabled", "score",
|
||||
}).AddRow("id1", "hello", "src", 0, "c1", "k1", "kb1", "t1", true, 0.8),
|
||||
)
|
||||
|
||||
results, err := repo.VectorRetrieve(context.Background(), types.RetrieveParams{
|
||||
Embedding: []float32{1, 2, 3},
|
||||
TopK: 5,
|
||||
Threshold: 0.5,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
require.Len(t, results[0].Results, 1)
|
||||
assert.Equal(t, "id1", results[0].Results[0].ID)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestNormalizeEmbedding(t *testing.T) {
|
||||
t.Run("returns unit vector copy", func(t *testing.T) {
|
||||
got := normalizeEmbedding([]float32{3, 4})
|
||||
require.Len(t, got, 2)
|
||||
assert.InDelta(t, 0.6, got[0], 1e-6)
|
||||
assert.InDelta(t, 0.8, got[1], 1e-6)
|
||||
})
|
||||
|
||||
t.Run("keeps zero vector finite", func(t *testing.T) {
|
||||
got := normalizeEmbedding([]float32{0, 0})
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, []float32{0, 0}, got)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKeywordsRetrieve_SQLShape(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
@@ -315,32 +368,44 @@ func TestKeywordsRetrieve_SQLShape(t *testing.T) {
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBatchUpdateChunkEnabledStatus_StreamLoad(t *testing.T) {
|
||||
// 这条路径覆盖了 lookupChunkRowKeys + partialUpdateRows 的连贯流。
|
||||
func TestBatchUpdateChunkEnabledStatus_RewritesRows(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
var streamLoadCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
streamLoadCalls++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Status": "Success",
|
||||
"NumberLoadedRows": 1,
|
||||
"Label": "label",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
httpClient: srv.Client(),
|
||||
feHTTPBase: srv.URL,
|
||||
username: "u",
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil)
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).AddRow("weknora_embeddings_768"))
|
||||
mock.ExpectQuery(`SELECT id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id, is_enabled, embedding FROM .*weknora_embeddings_768.* WHERE chunk_id IN`).
|
||||
WithArgs("c1").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"id", "content", "source_id", "source_type",
|
||||
"chunk_id", "knowledge_id", "knowledge_base_id", "tag_id",
|
||||
"is_enabled", "embedding",
|
||||
}).AddRow("row-1", "hello", "src1", 0, "c1", "k1", "kb1", "t1", true, "[1,2,3]"))
|
||||
mock.ExpectExec(`DELETE FROM .*weknora_embeddings_768.* WHERE id IN \(\?\)`).
|
||||
WithArgs("row-1").
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
mock.ExpectExec(`INSERT INTO .*weknora_embeddings_768.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`).
|
||||
WithArgs("row-1", "hello", "src1", 0, "c1", "k1", "kb1", "t1", false).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
require.NoError(t, repo.BatchUpdateChunkEnabledStatus(
|
||||
context.Background(), map[string]bool{"c1": false}))
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBatchUpdateChunkEnabledStatus_LegacyModeUsesPartialUpdate(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
primeCompatMode(repo, dorisCompatModeLegacy, nil)
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
@@ -351,7 +416,6 @@ func TestBatchUpdateChunkEnabledStatus_StreamLoad(t *testing.T) {
|
||||
|
||||
require.NoError(t, repo.BatchUpdateChunkEnabledStatus(
|
||||
context.Background(), map[string]bool{"c1": false}))
|
||||
assert.Equal(t, 1, streamLoadCalls, "stream load should be invoked once")
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
@@ -367,13 +431,14 @@ func TestEnsureTable_DDLShape(t *testing.T) {
|
||||
bucketsNum: 5,
|
||||
replicationNum: 2,
|
||||
}
|
||||
primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil)
|
||||
|
||||
// 表不存在
|
||||
mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings_768").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0))
|
||||
// CREATE TABLE 应包含关键属性
|
||||
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*UNIQUE KEY\(id\).*BUCKETS 5.*replication_num.*=.*2.*enable_unique_key_merge_on_write.*=.*true`).
|
||||
// CREATE TABLE 应包含关键属性和 Doris 支持的 inner_product ANN metric
|
||||
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*metric_type"="inner_product".*DUPLICATE KEY\(id\).*BUCKETS 5.*replication_num.*=.*2`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
// SHOW INDEX 一次即返回 ANN 已 FINISHED
|
||||
mock.ExpectQuery(`SHOW INDEX FROM .*weknora_embeddings_768.*`).
|
||||
@@ -389,6 +454,37 @@ func TestEnsureTable_DDLShape(t *testing.T) {
|
||||
}, 2*time.Second, 10*time.Millisecond, "expectations should be met after async ANN poll")
|
||||
}
|
||||
|
||||
func TestEnsureTable_DDLShape_LegacyMode(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
bucketsNum: 5,
|
||||
replicationNum: 2,
|
||||
}
|
||||
primeCompatMode(repo, dorisCompatModeLegacy, nil)
|
||||
|
||||
mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings_768").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0))
|
||||
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*metric_type"="cosine_distance".*UNIQUE KEY\(id\).*enable_unique_key_merge_on_write.*true`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectQuery(`SHOW INDEX FROM .*weknora_embeddings_768.*`).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"Table", "Key_name", "State"}).
|
||||
AddRow("weknora_embeddings_768", "idx_emb", "FINISHED"),
|
||||
)
|
||||
|
||||
require.NoError(t, repo.ensureTable(context.Background(), 768))
|
||||
require.Eventually(t, func() bool {
|
||||
return mock.ExpectationsWereMet() == nil
|
||||
}, 2*time.Second, 10*time.Millisecond, "expectations should be met after async ANN poll")
|
||||
}
|
||||
|
||||
func TestBatchSave_SQLShape(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
@@ -399,8 +495,52 @@ func TestBatchSave_SQLShape(t *testing.T) {
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil)
|
||||
repo.initializedTables.Store(3, true) // 跳过 ensureTable
|
||||
|
||||
mock.ExpectExec(`DELETE FROM .*weknora_embeddings_3.* WHERE id IN \(\?\)`).
|
||||
WithArgs("src1").
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
mock.ExpectExec(`INSERT INTO .*weknora_embeddings_3.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`).
|
||||
WithArgs(
|
||||
"src1", "hello", "src1", 0,
|
||||
"c1", "k1", "kb1", "",
|
||||
true,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
err = repo.BatchSave(context.Background(),
|
||||
[]*types.IndexInfo{{
|
||||
Content: "hello",
|
||||
SourceID: "src1",
|
||||
ChunkID: "c1",
|
||||
KnowledgeID: "k1",
|
||||
KnowledgeBaseID: "kb1",
|
||||
IsEnabled: true,
|
||||
}},
|
||||
map[string]any{
|
||||
fieldEmbedding: map[string][]float32{
|
||||
"src1": {1, 2, 3},
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBatchSave_SQLShape_LegacyMode(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
primeCompatMode(repo, dorisCompatModeLegacy, nil)
|
||||
repo.initializedTables.Store(3, true)
|
||||
|
||||
mock.ExpectExec(`INSERT INTO .*weknora_embeddings_3.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`).
|
||||
WithArgs(
|
||||
"src1", "hello", "src1", 0,
|
||||
@@ -428,6 +568,46 @@ func TestBatchSave_SQLShape(t *testing.T) {
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestResolveCompatMode_RejectsExistingModeSwitch(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
repo.compatModeRequested = dorisCompatModeLegacy
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).AddRow("weknora_embeddings_768"))
|
||||
mock.ExpectQuery(`SHOW CREATE TABLE .*weknora_embeddings_768.*`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"Table", "Create Table"}).AddRow(
|
||||
"weknora_embeddings_768",
|
||||
"CREATE TABLE `weknora_embeddings_768` (id VARCHAR(64)) ENGINE=OLAP DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 10 PROPERTIES(\"replication_num\"=\"1\")",
|
||||
))
|
||||
|
||||
_, err := repo.resolveCompatMode(context.Background())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not interchangeable after weknora_embeddings_*")
|
||||
assert.Contains(t, err.Error(), string(dorisCompatModeInnerProductDuplicate))
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestResolveCompatMode_AutoPrefersInnerProductDuplicate(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
repo.compatModeRequested = dorisCompatModeAuto
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}))
|
||||
mock.ExpectQuery(`SELECT inner_product_approximate\(\[1.0\],\[1.0\]\)`).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"v"}).AddRow(1.0))
|
||||
mock.ExpectQuery(`SELECT cosine_distance_approximate\(\[1.0\],\[1.0\]\)`).
|
||||
WillReturnError(errors.New("unsupported"))
|
||||
|
||||
mode, err := repo.resolveCompatMode(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, dorisCompatModeInnerProductDuplicate, mode)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Engine wiring
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -38,6 +38,10 @@ func (r *dorisRepository) ensureTable(ctx context.Context, dimension int) error
|
||||
if _, ok := r.initializedTables.Load(dimension); ok {
|
||||
return nil
|
||||
}
|
||||
compatMode, err := r.resolveCompatMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log := logger.GetLogger(ctx)
|
||||
tableName := r.getTableName(dimension)
|
||||
@@ -49,8 +53,8 @@ func (r *dorisRepository) ensureTable(ctx context.Context, dimension int) error
|
||||
}
|
||||
|
||||
if !exists {
|
||||
log.Infof("[Doris] Creating table %s with dimension %d", tableName, dimension)
|
||||
if err := r.createTable(ctx, tableName, dimension); err != nil {
|
||||
log.Infof("[Doris] Creating table %s with dimension %d in compat mode %s", tableName, dimension, compatMode)
|
||||
if err := r.createTable(ctx, tableName, dimension, compatMode); err != nil {
|
||||
log.Errorf("[Doris] Failed to create table: %v", err)
|
||||
return fmt.Errorf("create table: %w", err)
|
||||
}
|
||||
@@ -93,7 +97,7 @@ func (r *dorisRepository) tableExists(ctx context.Context, tableName string) (bo
|
||||
|
||||
// createTable 发出 CREATE TABLE DDL。Doris DDL 是同步的(除 ANN 索引构建外),
|
||||
// 返回成功即代表表已可写。
|
||||
func (r *dorisRepository) createTable(ctx context.Context, tableName string, dimension int) error {
|
||||
func (r *dorisRepository) createTable(ctx context.Context, tableName string, dimension int, compatMode dorisCompatMode) error {
|
||||
buckets := r.bucketsNum
|
||||
if buckets <= 0 {
|
||||
buckets = defaultBucketsNum
|
||||
@@ -103,22 +107,42 @@ func (r *dorisRepository) createTable(ctx context.Context, tableName string, dim
|
||||
replication = defaultReplicationNum
|
||||
}
|
||||
|
||||
ddl := buildCreateTableDDL(tableName, dimension, buckets, replication)
|
||||
ddl := buildCreateTableDDL(tableName, dimension, buckets, replication, compatMode)
|
||||
_, err := r.db.ExecContext(ctx, ddl)
|
||||
if err != nil && compatMode == dorisCompatModeLegacy {
|
||||
return fmt.Errorf(
|
||||
"legacy Doris table creation failed: %w. If your Doris build rejects ANN indexes on UNIQUE KEY tables, set %s=%s before creating embedding tables. %s is not interchangeable after %s_* tables are created",
|
||||
err,
|
||||
envDorisCompatMode,
|
||||
dorisCompatModeInnerProductDuplicate,
|
||||
envDorisCompatMode,
|
||||
r.tableBaseName,
|
||||
)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// buildCreateTableDDL 根据维度生成 CREATE TABLE DDL。
|
||||
//
|
||||
// 关键点:
|
||||
// - UNIQUE KEY(id) + enable_unique_key_merge_on_write=true:
|
||||
// 这是 partial update(Stream Load) 起作用的前提。
|
||||
// - DUPLICATE KEY(id):兼容当前 Doris/SelectDB 对 ANN 索引的表模型要求。
|
||||
// WeKnora 在 Go 端用 delete + insert 保持按 id 替换的写入语义。
|
||||
// - INVERTED 索引覆盖所有过滤字段 + 中文分词的 content 全文索引。
|
||||
// - ANN 索引使用 HNSW + cosine_distance,与 Qdrant/Weaviate 的 cosine 相似度方向一致。
|
||||
// - ANN 索引使用 HNSW + inner_product;Doris 写入/查询前会对向量单位化,
|
||||
// 因此整体仍保持与其他向量库一致的 cosine 相似度语义。
|
||||
//
|
||||
// 注意:DDL 中 dimension / buckets / replication 三个数值字段是 Go 端格式化拼接的,
|
||||
// 不存在 SQL 注入风险(来源都是受控的 IndexConfig int)。
|
||||
func buildCreateTableDDL(tableName string, dimension, buckets, replication int) string {
|
||||
func buildCreateTableDDL(tableName string, dimension, buckets, replication int, compatMode dorisCompatMode) string {
|
||||
metricType := "inner_product"
|
||||
keyMode := "DUPLICATE KEY(id)"
|
||||
properties := fmt.Sprintf("\t\"replication_num\"=\"%d\"", replication)
|
||||
if compatMode == dorisCompatModeLegacy {
|
||||
metricType = "cosine_distance"
|
||||
keyMode = "UNIQUE KEY(id)"
|
||||
properties = fmt.Sprintf("\t\"replication_num\"=\"%d\",\n\t\"enable_unique_key_merge_on_write\"=\"true\"", replication)
|
||||
}
|
||||
|
||||
const tpl = `CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
|
||||
id VARCHAR(64) NOT NULL,
|
||||
chunk_id VARCHAR(64),
|
||||
@@ -139,19 +163,18 @@ func buildCreateTableDDL(tableName string, dimension, buckets, replication int)
|
||||
INDEX idx_content (content) USING INVERTED PROPERTIES("parser"="chinese","support_phrase"="true"),
|
||||
INDEX idx_emb (embedding) USING ANN PROPERTIES(
|
||||
"index_type"="hnsw",
|
||||
"metric_type"="cosine_distance",
|
||||
"metric_type"="%s",
|
||||
"dim"="%d",
|
||||
"max_degree"="32",
|
||||
"ef_construction"="200"
|
||||
)
|
||||
) ENGINE=OLAP
|
||||
UNIQUE KEY(id)
|
||||
%s
|
||||
DISTRIBUTED BY HASH(id) BUCKETS %d
|
||||
PROPERTIES(
|
||||
"replication_num"="%d",
|
||||
"enable_unique_key_merge_on_write"="true"
|
||||
%s
|
||||
);`
|
||||
return fmt.Sprintf(tpl, tableName, dimension, buckets, replication)
|
||||
return fmt.Sprintf(tpl, tableName, metricType, dimension, keyMode, buckets, properties)
|
||||
}
|
||||
|
||||
// waitANNReady 轮询 SHOW INDEX,等待 ANN 索引进入 FINISHED 状态。
|
||||
|
||||
@@ -200,30 +200,117 @@ func chunkRows(rows []map[string]any, maxBytes int) [][]map[string]any {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// BatchUpdateChunkEnabledStatus 批量更新 chunk 的 is_enabled 字段。
|
||||
//
|
||||
// 由于 Stream Load partial update 需要主键列 id,但 chunk_id 不一定等于 id,
|
||||
// 这里做一个查表步骤:通过 chunk_id IN (...) 查出对应的 (id, table) 列表,
|
||||
// 再分表用 partial update 写回。
|
||||
//
|
||||
// 时间复杂度上是 O(N):扫描一遍 chunk_id IN (...) 在 INVERTED 索引下走点查。
|
||||
// legacy 模式走 Stream Load partial update;inner_product_duplicate 模式改为读整行后 replaceRows 写回。
|
||||
func (r *dorisRepository) BatchUpdateChunkEnabledStatus(ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
if len(chunkStatusMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
compatMode, err := r.resolveCompatMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !compatMode.usesRewriteChunkUpdates() {
|
||||
return r.batchUpdateChunkEnabledStatusLegacy(ctx, chunkStatusMap)
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, 0, len(chunkStatusMap))
|
||||
for id := range chunkStatusMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
|
||||
return r.rewriteChunkRows(ctx, chunkIDs, func(row *DorisVectorEmbedding) bool {
|
||||
enabled, ok := chunkStatusMap[row.ChunkID]
|
||||
if !ok || row.IsEnabled == enabled {
|
||||
return false
|
||||
}
|
||||
row.IsEnabled = enabled
|
||||
return true
|
||||
}, "rewrite is_enabled")
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID 批量更新 chunk 的 tag_id 字段。逻辑与 EnabledStatus 一致。
|
||||
func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
if len(chunkTagMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
compatMode, err := r.resolveCompatMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !compatMode.usesRewriteChunkUpdates() {
|
||||
return r.batchUpdateChunkTagIDLegacy(ctx, chunkTagMap)
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, 0, len(chunkTagMap))
|
||||
for id := range chunkTagMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
|
||||
return r.rewriteChunkRows(ctx, chunkIDs, func(row *DorisVectorEmbedding) bool {
|
||||
tagID, ok := chunkTagMap[row.ChunkID]
|
||||
if !ok || row.TagID == tagID {
|
||||
return false
|
||||
}
|
||||
row.TagID = tagID
|
||||
return true
|
||||
}, "rewrite tag_id")
|
||||
}
|
||||
|
||||
func (r *dorisRepository) rewriteChunkRows(ctx context.Context,
|
||||
chunkIDs []string,
|
||||
mutate func(*DorisVectorEmbedding) bool,
|
||||
action string,
|
||||
) error {
|
||||
if len(chunkIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tables, err := r.listEmbeddingTables(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list tables: %w", err)
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
rows, err := r.loadRowsByChunkIDs(ctx, table, chunkIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load chunk rows from %s: %w", table, err)
|
||||
}
|
||||
|
||||
updated := make([]*DorisVectorEmbedding, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !mutate(row) {
|
||||
continue
|
||||
}
|
||||
updated = append(updated, row)
|
||||
}
|
||||
if len(updated) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.replaceRows(ctx, table, updated); err != nil {
|
||||
return fmt.Errorf("%s in %s: %w", action, table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dorisRepository) batchUpdateChunkEnabledStatusLegacy(ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
chunkIDs := make([]string, 0, len(chunkStatusMap))
|
||||
for id := range chunkStatusMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
|
||||
mapping, err := r.lookupChunkRowKeys(ctx, chunkIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 按表分组 -> 每行 (id, is_enabled)。
|
||||
byTable := make(map[string][]map[string]any)
|
||||
for chunkID, locations := range mapping {
|
||||
enabled, ok := chunkStatusMap[chunkID]
|
||||
@@ -245,14 +332,9 @@ func (r *dorisRepository) BatchUpdateChunkEnabledStatus(ctx context.Context,
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID 批量更新 chunk 的 tag_id 字段。逻辑与 EnabledStatus 一致。
|
||||
func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context,
|
||||
func (r *dorisRepository) batchUpdateChunkTagIDLegacy(ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
if len(chunkTagMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, 0, len(chunkTagMap))
|
||||
for id := range chunkTagMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
@@ -284,6 +366,39 @@ func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *dorisRepository) loadRowsByChunkIDs(ctx context.Context,
|
||||
table string, chunkIDs []string,
|
||||
) ([]*DorisVectorEmbedding, error) {
|
||||
if len(chunkIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
placeholders := make([]string, len(chunkIDs))
|
||||
args := make([]any, len(chunkIDs))
|
||||
for i, v := range chunkIDs {
|
||||
placeholders[i] = "?"
|
||||
args[i] = v
|
||||
}
|
||||
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s FROM `%s` WHERE %s IN (%s)",
|
||||
strings.Join(columnsForCopy, ", "),
|
||||
table,
|
||||
fieldChunkID,
|
||||
strings.Join(placeholders, ", "),
|
||||
)
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
batch, err := scanCopyRows(rows)
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return batch, nil
|
||||
}
|
||||
|
||||
// rowLocation 表示某行在哪个表里、主键 id 是什么。
|
||||
type rowLocation struct {
|
||||
table string
|
||||
|
||||
@@ -10,10 +10,13 @@ import (
|
||||
//
|
||||
// 通信通道:
|
||||
// - 读写主链路:MySQL 协议(database/sql + go-sql-driver/mysql),FE 默认 9030 端口。
|
||||
// - Stream Load:HTTP(FE 默认 8030 端口),用于 BatchUpdate* 的 partial update。
|
||||
// - Stream Load:HTTP(FE 默认 8030 端口),legacy 模式用于 partial update。
|
||||
//
|
||||
// 表结构按维度分表:<tableBaseName>_<dim>,UNIQUE KEY(id),
|
||||
// 同时建立倒排索引(filter / 全文)+ ANN(HNSW) 索引。
|
||||
// 表结构按维度分表:<tableBaseName>_<dim>。
|
||||
// 兼容模式由 DORIS_COMPAT_MODE 决定:
|
||||
// - legacy:UNIQUE KEY(id) + cosine_distance ANN + Stream Load partial update
|
||||
// - inner_product_duplicate:DUPLICATE KEY(id) + normalized inner product + delete/insert rewrite
|
||||
// 该设置在 embedding 表创建后不可直接互换;切换模式前需要重建这些表。
|
||||
//
|
||||
// 与 Qdrant/Milvus/Weaviate 一样,initializedTables 缓存"已确保存在"的维度,
|
||||
// 避免每次写入都打 SHOW TABLES。
|
||||
@@ -32,6 +35,10 @@ type dorisRepository struct {
|
||||
tableBaseName string
|
||||
bucketsNum int // 0 -> default 10
|
||||
replicationNum int // 0 -> default 1
|
||||
compatModeRequested dorisCompatMode
|
||||
compatModeResolved dorisCompatMode
|
||||
compatResolveOnce sync.Once
|
||||
compatResolveErr error
|
||||
|
||||
// 已经确保过 ensureTable 的维度集合:dim -> true。
|
||||
initializedTables sync.Map
|
||||
@@ -55,8 +62,7 @@ type DorisVectorEmbedding struct {
|
||||
}
|
||||
|
||||
// DorisVectorEmbeddingWithScore 是检索结果的领域模型,
|
||||
// Score 在向量检索时存储 (1 - cosine_distance_approximate),
|
||||
// 在关键词检索时统一赋 1.0(与 Qdrant 行为一致)。
|
||||
// Score 在向量检索时按当前 compat mode 计算,在关键词检索时统一赋 1.0。
|
||||
type DorisVectorEmbeddingWithScore struct {
|
||||
DorisVectorEmbedding
|
||||
Score float64
|
||||
|
||||
Reference in New Issue
Block a user