e62c0563aa
Problem:
The hard-coded Doris vector function implementation (cosine_distance_approximate with
UNIQUE KEY ANN tables) fails on SelectDB 4.0.2-rc01 and other Doris builds lacking that
specific function support. Users had no way to adapt without code changes.
Root cause:
There is assumption all Doris deployments support the same vector function API, but different
builds (Doris OSS, SelectDB, Doris Cloud) ship with different function variants and table
key constraints. No capability detection or user configuration existed.
Solution:
Implement DORIS_COMPAT_MODE environment variable with three modes:
* auto (default/recommended): probe Doris server on first use to detect available vector
functions; prefer inner_product_duplicate (modern Doris 4.0+), fall back to legacy
(older builds lacking inner_product_approximate)
* legacy: hard-set to cosine_distance_approximate + UNIQUE KEY (for older Doris/SelectDB
builds without inner_product_approximate support)
* inner_product_duplicate: hard-set to inner_product_approximate + DUPLICATE KEY
(for modern Doris 4.0+ and current SelectDB with normalized embeddings)
Implementation details:
- add compat.go with one-time mode resolution (sync.Once) and capability probing
- inspect existing weknora_embeddings_* table DDL via SHOW CREATE TABLE to detect and
enforce schema compatibility; prevents silent mismatches
- fail fast with clear error message when configured mode does not match existing tables,
with explicit remediation steps (recreate tables or change env var)
- branch all query paths (inner_product_approximate vs cosine_distance_approximate),
DDL generation (DUPLICATE KEY vs UNIQUE KEY), write paths (embed normalization),
and chunk updates (Stream Load vs read-modify-write) by resolved compat mode
- add comprehensive repository tests for mode selection, auto-detection, and mismatch
scenarios; all tests pass
- expose DORIS_COMPAT_MODE in docker-compose.yml with auto as default
- document in .env.example with clear mode decision guidance
- log all mode decisions (requested, detected, probed, final) at INFO/WARN level
Key guarantee:
⚠️ DORIS_COMPAT_MODE is NOT interchangeable after embedding tables are created.
App will reject mode switches that conflict with existing table layout, preventing
silent data mismatches and query failures.
229 lines
6.7 KiB
Go
229 lines
6.7 KiB
Go
package doris
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
)
|
|
|
|
const envDorisCompatMode = "DORIS_COMPAT_MODE"
|
|
|
|
type dorisCompatMode string
|
|
|
|
const (
|
|
dorisCompatModeAuto dorisCompatMode = "auto"
|
|
dorisCompatModeLegacy dorisCompatMode = "legacy"
|
|
dorisCompatModeInnerProductDuplicate dorisCompatMode = "inner_product_duplicate"
|
|
)
|
|
|
|
type dorisCompatProbe struct {
|
|
innerProductApproximate bool
|
|
cosineDistanceApproximate bool
|
|
}
|
|
|
|
func resolveConfiguredDorisCompatMode() (mode dorisCompatMode, invalid string) {
|
|
raw := strings.TrimSpace(os.Getenv(envDorisCompatMode))
|
|
if raw == "" {
|
|
return dorisCompatModeAuto, ""
|
|
}
|
|
|
|
switch strings.ToLower(raw) {
|
|
case string(dorisCompatModeAuto):
|
|
return dorisCompatModeAuto, ""
|
|
case string(dorisCompatModeLegacy):
|
|
return dorisCompatModeLegacy, ""
|
|
case string(dorisCompatModeInnerProductDuplicate), "inner-product-duplicate", "inner_product", "inner-product":
|
|
return dorisCompatModeInnerProductDuplicate, ""
|
|
default:
|
|
return dorisCompatModeAuto, raw
|
|
}
|
|
}
|
|
|
|
func (r *dorisRepository) resolveCompatMode(ctx context.Context) (dorisCompatMode, error) {
|
|
r.compatResolveOnce.Do(func() {
|
|
log := logger.GetLogger(ctx)
|
|
requested := r.compatModeRequested
|
|
if requested == "" {
|
|
r.compatModeResolved = dorisCompatModeInnerProductDuplicate
|
|
return
|
|
}
|
|
|
|
existingMode, exampleTable, foundExisting, err := r.detectExistingCompatMode(ctx)
|
|
if err != nil {
|
|
r.compatResolveErr = err
|
|
return
|
|
}
|
|
|
|
if foundExisting {
|
|
if requested != dorisCompatModeAuto && requested != existingMode {
|
|
r.compatResolveErr = fmt.Errorf(
|
|
"Doris compat mode %q does not match existing embedding tables (detected %q from %s). %s is not interchangeable after %s_* tables are created. Recreate the existing %s_* tables before switching modes, or set %s=%s",
|
|
requested,
|
|
existingMode,
|
|
exampleTable,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
r.tableBaseName,
|
|
envDorisCompatMode,
|
|
existingMode,
|
|
)
|
|
log.Errorf("[Doris] %v", r.compatResolveErr)
|
|
return
|
|
}
|
|
|
|
r.compatModeResolved = existingMode
|
|
log.Warnf(
|
|
"[Doris] Using compat mode %s from existing embedding tables (%s). %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
|
existingMode,
|
|
exampleTable,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
)
|
|
return
|
|
}
|
|
|
|
if requested == dorisCompatModeAuto {
|
|
probe, probeErr := r.probeCompatMode(ctx)
|
|
if probeErr != nil {
|
|
r.compatResolveErr = probeErr
|
|
log.Errorf("[Doris] %v", probeErr)
|
|
return
|
|
}
|
|
log.Infof(
|
|
"[Doris] Compat probe result: inner_product_approximate=%t, cosine_distance_approximate=%t",
|
|
probe.innerProductApproximate,
|
|
probe.cosineDistanceApproximate,
|
|
)
|
|
|
|
switch {
|
|
case probe.innerProductApproximate:
|
|
r.compatModeResolved = dorisCompatModeInnerProductDuplicate
|
|
case probe.cosineDistanceApproximate:
|
|
r.compatModeResolved = dorisCompatModeLegacy
|
|
default:
|
|
r.compatResolveErr = fmt.Errorf(
|
|
"Doris compatibility auto-detection could not find a supported vector function. Set %s=%s or %s=%s explicitly after verifying your Doris build. %s is not interchangeable after %s_* tables are created",
|
|
envDorisCompatMode,
|
|
dorisCompatModeInnerProductDuplicate,
|
|
envDorisCompatMode,
|
|
dorisCompatModeLegacy,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
)
|
|
log.Errorf("[Doris] %v", r.compatResolveErr)
|
|
return
|
|
}
|
|
|
|
log.Warnf(
|
|
"[Doris] Auto-selected compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
|
r.compatModeResolved,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
)
|
|
return
|
|
}
|
|
|
|
r.compatModeResolved = requested
|
|
log.Warnf(
|
|
"[Doris] Using configured compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.",
|
|
r.compatModeResolved,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
)
|
|
})
|
|
|
|
return r.compatModeResolved, r.compatResolveErr
|
|
}
|
|
|
|
func (r *dorisRepository) detectExistingCompatMode(ctx context.Context) (dorisCompatMode, string, bool, error) {
|
|
tables, err := r.listEmbeddingTables(ctx)
|
|
if err != nil {
|
|
return "", "", false, fmt.Errorf("list Doris embedding tables: %w", err)
|
|
}
|
|
if len(tables) == 0 {
|
|
return "", "", false, nil
|
|
}
|
|
sort.Strings(tables)
|
|
|
|
var detected dorisCompatMode
|
|
for _, table := range tables {
|
|
ddl, err := r.showCreateTable(ctx, table)
|
|
if err != nil {
|
|
return "", "", false, fmt.Errorf("show create table %s: %w", table, err)
|
|
}
|
|
mode, err := compatModeFromDDL(ddl)
|
|
if err != nil {
|
|
return "", "", false, fmt.Errorf("detect compat mode from %s: %w", table, err)
|
|
}
|
|
if detected == "" {
|
|
detected = mode
|
|
continue
|
|
}
|
|
if detected != mode {
|
|
return "", "", false, fmt.Errorf(
|
|
"existing Doris embedding tables use mixed compat modes (%s and %s). %s is not interchangeable after %s_* tables are created; recreate the existing %s_* tables with a single mode",
|
|
detected,
|
|
mode,
|
|
envDorisCompatMode,
|
|
r.tableBaseName,
|
|
r.tableBaseName,
|
|
)
|
|
}
|
|
}
|
|
|
|
return detected, tables[0], true, nil
|
|
}
|
|
|
|
func (r *dorisRepository) showCreateTable(ctx context.Context, table string) (string, error) {
|
|
row := r.db.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`", table))
|
|
var tableName, ddl string
|
|
if err := row.Scan(&tableName, &ddl); err != nil {
|
|
return "", err
|
|
}
|
|
return ddl, nil
|
|
}
|
|
|
|
func compatModeFromDDL(ddl string) (dorisCompatMode, error) {
|
|
normalized := strings.ToLower(ddl)
|
|
switch {
|
|
case strings.Contains(normalized, "duplicate key("):
|
|
return dorisCompatModeInnerProductDuplicate, nil
|
|
case strings.Contains(normalized, "unique key("):
|
|
return dorisCompatModeLegacy, nil
|
|
default:
|
|
return "", fmt.Errorf("unsupported table definition; expected UNIQUE KEY or DUPLICATE KEY")
|
|
}
|
|
}
|
|
|
|
func (r *dorisRepository) probeCompatMode(ctx context.Context) (dorisCompatProbe, error) {
|
|
probe := dorisCompatProbe{}
|
|
probe.innerProductApproximate = r.vectorFunctionSupported(ctx, "inner_product_approximate([1.0],[1.0])")
|
|
probe.cosineDistanceApproximate = r.vectorFunctionSupported(ctx, "cosine_distance_approximate([1.0],[1.0])")
|
|
return probe, nil
|
|
}
|
|
|
|
func (r *dorisRepository) vectorFunctionSupported(ctx context.Context, expr string) bool {
|
|
var value sql.NullFloat64
|
|
if err := r.db.QueryRowContext(ctx, "SELECT "+expr).Scan(&value); err != nil {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (m dorisCompatMode) normalizeEmbeddings() bool {
|
|
return m != dorisCompatModeLegacy
|
|
}
|
|
|
|
func (m dorisCompatMode) usesReplaceWrite() bool {
|
|
return m != dorisCompatModeLegacy
|
|
}
|
|
|
|
func (m dorisCompatMode) usesRewriteChunkUpdates() bool {
|
|
return m != dorisCompatModeLegacy
|
|
}
|