feat(retriever): 接入 Apache Doris 4.1 作为向量数据库
为 RetrieveEngine 体系新增 Doris 后端,与现有 Qdrant/Milvus/Weaviate
等保持完整能力对齐:向量检索、关键词检索、健康检查、环境变量与多实例
DB 配置、前端类型注册、单元测试、Docker Compose 模板。
实现要点:
- 协议分工:主链路用 MySQL 协议(database/sql + go-sql-driver/mysql)
做 DDL / 查询 / 删除;批量更新走 Stream Load HTTP API,并启用
partial_update=true、merge_type=APPEND,自动按 1MiB 切分批次并处理
307 重定向。
- 表结构:UNIQUE KEY(id) + enable_unique_key_merge_on_write=true 以
支持 upsert/部分列更新;按维度分表(<base>_<dim>),每张表上建
HNSW ANN 索引(metric_type=cosine_distance)和 INVERTED 索引
(parser=chinese)。
- 分数语义:使用 cosine_distance_approximate,再以 1 - dist 转换为
"越大越相似",与现有 KVHybridRetrieveEngine 约定一致。
- 异步索引:ANN 索引为后台构建,ensureTable 通过轮询 SHOW INDEX 等
待索引就绪后再放行写入,避免首次检索召回为空。
- ARRAY<FLOAT> 序列化:go-sql-driver/mysql 不支持数组占位符,
embeddingLiteral 将 []float32 转成 SQL 字面量字符串再拼接。
新增文件:
- internal/application/repository/retriever/doris/{structs,schema,
query,repository,streamload,repository_test}.go
- scripts/e2e-doris.sh:E2E 验证清单
- docs/wiki/集成扩展/Doris改动与上游同步.md:fork-and-rebase 工作流
与改动清单
修改文件(接线 + 文档):
- internal/types/{retriever,tenant,vectorstore}.go:新增
DorisRetrieverEngineType、env 解析、表单 schema 与索引参数校验
- internal/container/{container,engine_factory}.go:环境变量驱动
与 VectorStore 配置驱动两条路径都支持 Doris
- internal/application/service/vectorstore{,_healthcheck}.go:连接
校验 + Ping/Version 健康检查
- docker-compose.yml:新增 doris-fe / doris-be 服务(profile=doris)
- .env.example:DORIS_* 环境变量与示例
- docs/{使用其他向量数据库,wiki/集成扩展/集成向量数据库}.md:
使用说明与索引/分数行为说明
依赖:go.mod/go.sum 新增 github.com/go-sql-driver/mysql(运行时)和
github.com/DATA-DOG/go-sqlmock(测试)。
测试:repository 层 SQL 形状、Stream Load HTTP 行为、whereBuilder
逻辑、embeddingLiteral 往返、健康检查错误路径均有单测覆盖。
This commit is contained in:
@@ -12,6 +12,7 @@ const (
|
||||
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
|
||||
MilvusRetrieverEngineType RetrieverEngineType = "milvus"
|
||||
WeaviateRetrieverEngineType RetrieverEngineType = "weaviate"
|
||||
DorisRetrieverEngineType RetrieverEngineType = "doris"
|
||||
SQLiteRetrieverEngineType RetrieverEngineType = "sqlite"
|
||||
)
|
||||
|
||||
|
||||
@@ -37,6 +37,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: WeaviateRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: WeaviateRetrieverEngineType},
|
||||
},
|
||||
"doris": {
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: DorisRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: DorisRetrieverEngineType},
|
||||
},
|
||||
"sqlite": {
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -72,6 +73,7 @@ var validEngineTypes = map[RetrieverEngineType]bool{
|
||||
QdrantRetrieverEngineType: true,
|
||||
MilvusRetrieverEngineType: true,
|
||||
WeaviateRetrieverEngineType: true,
|
||||
DorisRetrieverEngineType: true,
|
||||
SQLiteRetrieverEngineType: true,
|
||||
}
|
||||
|
||||
@@ -115,6 +117,12 @@ type ConnectionConfig struct {
|
||||
Scheme string `yaml:"scheme" json:"scheme,omitempty"`
|
||||
// Postgres
|
||||
UseDefaultConnection bool `yaml:"use_default_connection" json:"use_default_connection,omitempty"`
|
||||
// Doris: HTTP port for Stream Load API (FE default 8030).
|
||||
// Addr is reused for the MySQL protocol "host:9030"; HTTPPort + the host of Addr
|
||||
// together form the FE HTTP endpoint used by Stream Load.
|
||||
HTTPPort int `yaml:"http_port" json:"http_port,omitempty"`
|
||||
// Doris: target database name for the Stream Load HTTP path and the MySQL DSN.
|
||||
Database string `yaml:"database" json:"database,omitempty"`
|
||||
// Version is the detected server version (e.g., "7.10.1", "16.2", "1.12.6").
|
||||
// Auto-populated by TestConnection on successful connectivity check.
|
||||
Version string `yaml:"version" json:"version,omitempty"`
|
||||
@@ -214,6 +222,8 @@ type IndexConfig struct {
|
||||
ShardsNum int `yaml:"shards_num" json:"shards_num,omitempty"` // Milvus: number of shards per collection (CreateCollection)
|
||||
ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus: in-memory replica count (LoadCollection)
|
||||
DesiredShardCount int `yaml:"desired_shard_count" json:"desired_shard_count,omitempty"` // Weaviate: number of shards per collection
|
||||
BucketsNum int `yaml:"buckets_num" json:"buckets_num,omitempty"` // Doris: number of buckets per table (DISTRIBUTED BY HASH ... BUCKETS N)
|
||||
ReplicationNum int `yaml:"replication_num" json:"replication_num,omitempty"` // Doris: replication_num PROPERTIES
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
@@ -257,6 +267,16 @@ func (c IndexConfig) GetIndexNameOrDefault(engineType RetrieverEngineType) strin
|
||||
return c.CollectionPrefix
|
||||
}
|
||||
return "Weknora_embeddings"
|
||||
case DorisRetrieverEngineType:
|
||||
// Doris uses the prefix as the table base name; per-dimension tables are
|
||||
// suffixed with _<dim> at runtime by the repository layer.
|
||||
if c.CollectionPrefix != "" {
|
||||
return c.CollectionPrefix
|
||||
}
|
||||
if c.CollectionName != "" {
|
||||
return c.CollectionName
|
||||
}
|
||||
return "weknora_embeddings"
|
||||
default:
|
||||
return c.IndexName
|
||||
}
|
||||
@@ -327,6 +347,22 @@ func (c *IndexConfig) GetDesiredShardCount(def int) int {
|
||||
return def
|
||||
}
|
||||
|
||||
// GetBucketsNum returns the configured buckets_num (Doris), or def if unset/zero.
|
||||
func (c *IndexConfig) GetBucketsNum(def int) int {
|
||||
if c != nil && c.BucketsNum > 0 {
|
||||
return c.BucketsNum
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// GetReplicationNum returns the configured replication_num (Doris), or def if unset/zero.
|
||||
func (c *IndexConfig) GetReplicationNum(def int) int {
|
||||
if c != nil && c.ReplicationNum > 0 {
|
||||
return c.ReplicationNum
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IndexConfig — resolve helpers (for Repository layer, with env var fallback)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -427,6 +463,12 @@ func ValidateIndexConfig(ic IndexConfig) error {
|
||||
if ic.DesiredShardCount < 0 || ic.DesiredShardCount > maxShards {
|
||||
return errors.NewValidationError(fmt.Sprintf("desired_shard_count must be between 0 and %d", maxShards))
|
||||
}
|
||||
if ic.BucketsNum < 0 || ic.BucketsNum > maxShards {
|
||||
return errors.NewValidationError(fmt.Sprintf("buckets_num must be between 0 and %d", maxShards))
|
||||
}
|
||||
if ic.ReplicationNum < 0 || ic.ReplicationNum > maxReplicas {
|
||||
return errors.NewValidationError(fmt.Sprintf("replication_num must be between 0 and %d", maxReplicas))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -541,6 +583,22 @@ func GetVectorStoreTypes() []VectorStoreTypeInfo {
|
||||
{Name: "replication_factor", Type: "number", Required: false, Description: "Replication Factor", Default: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "doris",
|
||||
DisplayName: "Apache Doris",
|
||||
ConnectionFields: []VectorStoreFieldInfo{
|
||||
{Name: "addr", Type: "string", Required: true, Description: "FE MySQL Address (host:port)", Default: "doris-fe:9030"},
|
||||
{Name: "http_port", Type: "number", Required: false, Description: "FE HTTP Port (Stream Load)", Default: 8030},
|
||||
{Name: "database", Type: "string", Required: true, Description: "Database", Default: "weknora"},
|
||||
{Name: "username", Type: "string", Required: false, Description: "Username", Default: "root"},
|
||||
{Name: "password", Type: "string", Required: false, Sensitive: true, Description: "Password"},
|
||||
},
|
||||
IndexFields: []VectorStoreFieldInfo{
|
||||
{Name: "collection_prefix", Type: "string", Required: false, Description: "Table Prefix", Default: "weknora_embeddings"},
|
||||
{Name: "buckets_num", Type: "number", Required: false, Description: "Buckets per table", Default: 10},
|
||||
{Name: "replication_num", Type: "number", Required: false, Description: "Replication Num", Default: 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -668,6 +726,28 @@ func buildEnvStoreForDriver(driver string, envLookup EnvLookupFunc) *VectorStore
|
||||
APIKey: envLookup("WEAVIATE_API_KEY"),
|
||||
},
|
||||
}
|
||||
case "doris":
|
||||
httpPort := 0
|
||||
if v := envLookup("DORIS_HTTP_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil {
|
||||
httpPort = p
|
||||
}
|
||||
}
|
||||
return &VectorStore{
|
||||
ID: "__env_doris__",
|
||||
Name: "Apache Doris",
|
||||
EngineType: DorisRetrieverEngineType,
|
||||
ConnectionConfig: ConnectionConfig{
|
||||
Addr: envLookup("DORIS_ADDR"),
|
||||
HTTPPort: httpPort,
|
||||
Database: envLookup("DORIS_DATABASE"),
|
||||
Username: envLookup("DORIS_USERNAME"),
|
||||
Password: envLookup("DORIS_PASSWORD"),
|
||||
},
|
||||
IndexConfig: IndexConfig{
|
||||
CollectionPrefix: envLookup("DORIS_TABLE_PREFIX"),
|
||||
},
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -51,6 +51,12 @@ func TestBuildEnvVectorStores(t *testing.T) {
|
||||
"QDRANT_API_KEY": "qd-key",
|
||||
"MILVUS_ADDRESS": "milvus:19530",
|
||||
"WEAVIATE_HOST": "weaviate:8080",
|
||||
"DORIS_ADDR": "doris-fe:9030",
|
||||
"DORIS_HTTP_PORT": "8030",
|
||||
"DORIS_DATABASE": "weknora",
|
||||
"DORIS_USERNAME": "root",
|
||||
"DORIS_PASSWORD": "doris-pass",
|
||||
"DORIS_TABLE_PREFIX": "weknora_embeddings",
|
||||
}
|
||||
lookup := mockEnvLookup(envMap)
|
||||
|
||||
@@ -97,8 +103,8 @@ func TestBuildEnvVectorStores(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("all supported drivers", func(t *testing.T) {
|
||||
stores := BuildEnvVectorStores("postgres,sqlite,elasticsearch_v8,elasticsearch_v7,qdrant,milvus,weaviate", lookup)
|
||||
require.Len(t, stores, 7)
|
||||
stores := BuildEnvVectorStores("postgres,sqlite,elasticsearch_v8,elasticsearch_v7,qdrant,milvus,weaviate,doris", lookup)
|
||||
require.Len(t, stores, 8)
|
||||
|
||||
ids := make([]string, len(stores))
|
||||
for i, s := range stores {
|
||||
@@ -111,6 +117,7 @@ func TestBuildEnvVectorStores(t *testing.T) {
|
||||
assert.Contains(t, ids, "__env_qdrant__")
|
||||
assert.Contains(t, ids, "__env_milvus__")
|
||||
assert.Contains(t, ids, "__env_weaviate__")
|
||||
assert.Contains(t, ids, "__env_doris__")
|
||||
})
|
||||
|
||||
t.Run("qdrant env store", func(t *testing.T) {
|
||||
@@ -131,6 +138,30 @@ func TestBuildEnvVectorStores(t *testing.T) {
|
||||
require.Len(t, stores, 1)
|
||||
assert.Equal(t, "weaviate:8080", stores[0].ConnectionConfig.Host)
|
||||
})
|
||||
|
||||
t.Run("doris env store", func(t *testing.T) {
|
||||
stores := BuildEnvVectorStores("doris", lookup)
|
||||
require.Len(t, stores, 1)
|
||||
assert.Equal(t, "__env_doris__", stores[0].ID)
|
||||
assert.Equal(t, DorisRetrieverEngineType, stores[0].EngineType)
|
||||
assert.Equal(t, "doris-fe:9030", stores[0].ConnectionConfig.Addr)
|
||||
assert.Equal(t, 8030, stores[0].ConnectionConfig.HTTPPort)
|
||||
assert.Equal(t, "weknora", stores[0].ConnectionConfig.Database)
|
||||
assert.Equal(t, "root", stores[0].ConnectionConfig.Username)
|
||||
assert.Equal(t, "doris-pass", stores[0].ConnectionConfig.Password)
|
||||
assert.Equal(t, "weknora_embeddings", stores[0].IndexConfig.CollectionPrefix)
|
||||
})
|
||||
|
||||
t.Run("doris env store handles invalid http port gracefully", func(t *testing.T) {
|
||||
bad := mockEnvLookup(map[string]string{
|
||||
"DORIS_ADDR": "doris-fe:9030",
|
||||
"DORIS_HTTP_PORT": "not-a-number",
|
||||
"DORIS_DATABASE": "weknora",
|
||||
})
|
||||
stores := BuildEnvVectorStores("doris", bad)
|
||||
require.Len(t, stores, 1)
|
||||
assert.Equal(t, 0, stores[0].ConnectionConfig.HTTPPort) // falls back to 0 (factory will default to 8030)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFindEnvVectorStore(t *testing.T) {
|
||||
@@ -198,8 +229,8 @@ func TestNewVectorStoreResponse(t *testing.T) {
|
||||
func TestGetVectorStoreTypes(t *testing.T) {
|
||||
types := GetVectorStoreTypes()
|
||||
|
||||
t.Run("returns 4 engine types (excludes postgres and sqlite)", func(t *testing.T) {
|
||||
assert.Len(t, types, 4)
|
||||
t.Run("returns 5 engine types (excludes postgres and sqlite)", func(t *testing.T) {
|
||||
assert.Len(t, types, 5)
|
||||
})
|
||||
|
||||
t.Run("type names match engine constants", func(t *testing.T) {
|
||||
@@ -211,10 +242,32 @@ func TestGetVectorStoreTypes(t *testing.T) {
|
||||
assert.Contains(t, typeNames, "qdrant")
|
||||
assert.Contains(t, typeNames, "milvus")
|
||||
assert.Contains(t, typeNames, "weaviate")
|
||||
assert.Contains(t, typeNames, "doris")
|
||||
assert.NotContains(t, typeNames, "postgres")
|
||||
assert.NotContains(t, typeNames, "sqlite")
|
||||
})
|
||||
|
||||
t.Run("doris has connection and index fields", func(t *testing.T) {
|
||||
var dorisType VectorStoreTypeInfo
|
||||
for _, typ := range types {
|
||||
if typ.Type == "doris" {
|
||||
dorisType = typ
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, dorisType.ConnectionFields)
|
||||
require.NotEmpty(t, dorisType.IndexFields)
|
||||
|
||||
// addr and database are required
|
||||
seen := map[string]VectorStoreFieldInfo{}
|
||||
for _, f := range dorisType.ConnectionFields {
|
||||
seen[f.Name] = f
|
||||
}
|
||||
assert.True(t, seen["addr"].Required)
|
||||
assert.True(t, seen["database"].Required)
|
||||
assert.True(t, seen["password"].Sensitive)
|
||||
})
|
||||
|
||||
t.Run("elasticsearch has connection and index fields", func(t *testing.T) {
|
||||
var esType VectorStoreTypeInfo
|
||||
for _, typ := range types {
|
||||
@@ -898,6 +951,35 @@ func TestValidateIndexConfig(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "desired_shard_count")
|
||||
})
|
||||
|
||||
t.Run("buckets_num exceeds max", func(t *testing.T) {
|
||||
ic := IndexConfig{BucketsNum: 999}
|
||||
err := ValidateIndexConfig(ic)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "buckets_num")
|
||||
})
|
||||
|
||||
t.Run("buckets_num at max boundary is valid", func(t *testing.T) {
|
||||
ic := IndexConfig{BucketsNum: 64}
|
||||
assert.NoError(t, ValidateIndexConfig(ic))
|
||||
})
|
||||
|
||||
t.Run("replication_num exceeds max", func(t *testing.T) {
|
||||
ic := IndexConfig{ReplicationNum: 50}
|
||||
err := ValidateIndexConfig(ic)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "replication_num")
|
||||
})
|
||||
|
||||
t.Run("doris GetIndexNameOrDefault falls back when prefix empty", func(t *testing.T) {
|
||||
ic := IndexConfig{}
|
||||
assert.Equal(t, "weknora_embeddings", ic.GetIndexNameOrDefault(DorisRetrieverEngineType))
|
||||
})
|
||||
|
||||
t.Run("doris GetIndexNameOrDefault honors collection_prefix", func(t *testing.T) {
|
||||
ic := IndexConfig{CollectionPrefix: "custom_prefix"}
|
||||
assert.Equal(t, "custom_prefix", ic.GetIndexNameOrDefault(DorisRetrieverEngineType))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user