feat(retriever): 接入 Apache Doris 4.1 作为向量数据库
为 RetrieveEngine 体系新增 Doris 后端,与现有 Qdrant/Milvus/Weaviate
等保持完整能力对齐:向量检索、关键词检索、健康检查、环境变量与多实例
DB 配置、前端类型注册、单元测试、Docker Compose 模板。
实现要点:
- 协议分工:主链路用 MySQL 协议(database/sql + go-sql-driver/mysql)
做 DDL / 查询 / 删除;批量更新走 Stream Load HTTP API,并启用
partial_update=true、merge_type=APPEND,自动按 1MiB 切分批次并处理
307 重定向。
- 表结构:UNIQUE KEY(id) + enable_unique_key_merge_on_write=true 以
支持 upsert/部分列更新;按维度分表(<base>_<dim>),每张表上建
HNSW ANN 索引(metric_type=cosine_distance)和 INVERTED 索引
(parser=chinese)。
- 分数语义:使用 cosine_distance_approximate,再以 1 - dist 转换为
"越大越相似",与现有 KVHybridRetrieveEngine 约定一致。
- 异步索引:ANN 索引为后台构建,ensureTable 通过轮询 SHOW INDEX 等
待索引就绪后再放行写入,避免首次检索召回为空。
- ARRAY<FLOAT> 序列化:go-sql-driver/mysql 不支持数组占位符,
embeddingLiteral 将 []float32 转成 SQL 字面量字符串再拼接。
新增文件:
- internal/application/repository/retriever/doris/{structs,schema,
query,repository,streamload,repository_test}.go
- scripts/e2e-doris.sh:E2E 验证清单
- docs/wiki/集成扩展/Doris改动与上游同步.md:fork-and-rebase 工作流
与改动清单
修改文件(接线 + 文档):
- internal/types/{retriever,tenant,vectorstore}.go:新增
DorisRetrieverEngineType、env 解析、表单 schema 与索引参数校验
- internal/container/{container,engine_factory}.go:环境变量驱动
与 VectorStore 配置驱动两条路径都支持 Doris
- internal/application/service/vectorstore{,_healthcheck}.go:连接
校验 + Ping/Version 健康检查
- docker-compose.yml:新增 doris-fe / doris-be 服务(profile=doris)
- .env.example:DORIS_* 环境变量与示例
- docs/{使用其他向量数据库,wiki/集成扩展/集成向量数据库}.md:
使用说明与索引/分数行为说明
依赖:go.mod/go.sum 新增 github.com/go-sql-driver/mysql(运行时)和
github.com/DATA-DOG/go-sqlmock(测试)。
测试:repository 层 SQL 形状、Stream Load HTTP 行为、whereBuilder
逻辑、embeddingLiteral 往返、健康检查错误路径均有单测覆盖。
This commit is contained in:
@@ -20,6 +20,7 @@ import (
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
_ "github.com/go-sql-driver/mysql" // 给 Doris (database/sql) 注册 MySQL 协议驱动
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
@@ -33,6 +34,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
|
||||
dorisRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/doris"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
milvusRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/milvus"
|
||||
@@ -954,6 +956,53 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "doris") {
|
||||
dorisAddr := os.Getenv("DORIS_ADDR")
|
||||
if dorisAddr == "" {
|
||||
// docker-compose 默认服务名 + Doris FE MySQL 端口
|
||||
dorisAddr = "doris-fe:9030"
|
||||
}
|
||||
dorisDatabase := os.Getenv("DORIS_DATABASE")
|
||||
if dorisDatabase == "" {
|
||||
dorisDatabase = "weknora"
|
||||
}
|
||||
dorisUsername := os.Getenv("DORIS_USERNAME")
|
||||
if dorisUsername == "" {
|
||||
dorisUsername = "root"
|
||||
}
|
||||
dorisPassword := os.Getenv("DORIS_PASSWORD")
|
||||
dorisHTTPPort := 8030
|
||||
if portStr := os.Getenv("DORIS_HTTP_PORT"); portStr != "" {
|
||||
if port, err := strconv.Atoi(portStr); err == nil {
|
||||
dorisHTTPPort = port
|
||||
}
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",
|
||||
dorisUsername, dorisPassword, dorisAddr, dorisDatabase)
|
||||
dorisDB, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
log.Errorf("Create doris client failed: %v", err)
|
||||
} else {
|
||||
dorisDB.SetMaxOpenConns(20)
|
||||
dorisDB.SetMaxIdleConns(5)
|
||||
dorisDB.SetConnMaxLifetime(time.Hour)
|
||||
|
||||
httpBase := "http://" + hostFromAddr(dorisAddr) + ":" + strconv.Itoa(dorisHTTPPort)
|
||||
dorisRepository := dorisRepo.NewDorisRetrieveEngineRepository(
|
||||
dorisDB, httpBase, dorisUsername, dorisPassword, dorisDatabase, nil,
|
||||
)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(
|
||||
dorisRepository, types.DorisRetrieverEngineType,
|
||||
),
|
||||
); err != nil {
|
||||
log.Errorf("Register doris retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register doris retrieve engine success: %s db=%s", dorisAddr, dorisDatabase)
|
||||
}
|
||||
}
|
||||
}
|
||||
// ─── DB store registration (byStoreID) ───
|
||||
if storeReg, ok := registry.(*retriever.RetrieveEngineRegistry); ok {
|
||||
loadDBStoresIntoRegistry(storeReg, db, cfg)
|
||||
|
||||
@@ -2,12 +2,15 @@ package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
_ "github.com/go-sql-driver/mysql" // 通过 database/sql 注册 mysql 驱动给 Doris 使用
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate"
|
||||
@@ -16,6 +19,7 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
"gorm.io/gorm"
|
||||
|
||||
dorisRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/doris"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
milvusRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/milvus"
|
||||
@@ -56,6 +60,8 @@ func createEngineServiceFromStore(
|
||||
return createMilvusEngine(ctx, store)
|
||||
case types.WeaviateRetrieverEngineType:
|
||||
return createWeaviateEngine(store)
|
||||
case types.DorisRetrieverEngineType:
|
||||
return createDorisEngine(store)
|
||||
case types.SQLiteRetrieverEngineType:
|
||||
return createSQLiteEngine(store, db)
|
||||
default:
|
||||
@@ -205,3 +211,49 @@ func createWeaviateEngine(store types.VectorStore) (interfaces.RetrieveEngineSer
|
||||
repo := weaviateRepo.NewWeaviateRetrieveEngineRepository(client, &store.IndexConfig)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.WeaviateRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
// createDorisEngine 创建 Apache Doris 检索引擎服务。
|
||||
//
|
||||
// Doris 同时使用两个端口:
|
||||
// - MySQL 协议(默认 9030)走 database/sql 做主链路读写;
|
||||
// - HTTP(默认 FE 8030)走 Stream Load 做 partial update。
|
||||
//
|
||||
// Addr 字段承担 host:9030 的 MySQL 端点;HTTPPort + Addr 的 host 部分组成 HTTP base URL。
|
||||
func createDorisEngine(store types.VectorStore) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
if cc.Addr == "" {
|
||||
return nil, fmt.Errorf("doris connection requires addr (host:port)")
|
||||
}
|
||||
if cc.Database == "" {
|
||||
return nil, fmt.Errorf("doris connection requires database")
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=true&loc=Local",
|
||||
cc.Username, cc.Password, cc.Addr, cc.Database)
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create doris client: %w", err)
|
||||
}
|
||||
db.SetMaxOpenConns(20)
|
||||
db.SetMaxIdleConns(5)
|
||||
db.SetConnMaxLifetime(time.Hour)
|
||||
|
||||
httpPort := cc.HTTPPort
|
||||
if httpPort <= 0 {
|
||||
httpPort = 8030
|
||||
}
|
||||
httpBase := "http://" + hostFromAddr(cc.Addr) + ":" + strconv.Itoa(httpPort)
|
||||
|
||||
repo := dorisRepo.NewDorisRetrieveEngineRepository(
|
||||
db, httpBase, cc.Username, cc.Password, cc.Database, &store.IndexConfig,
|
||||
)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.DorisRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
// hostFromAddr 从 "host:port" 中拆出 host 部分;Addr 没有冒号时整段当作 host。
|
||||
func hostFromAddr(addr string) string {
|
||||
if i := strings.LastIndex(addr, ":"); i > 0 {
|
||||
return addr[:i]
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user