4cce6f2e99
为 RetrieveEngine 体系新增 Doris 后端,与现有 Qdrant/Milvus/Weaviate
等保持完整能力对齐:向量检索、关键词检索、健康检查、环境变量与多实例
DB 配置、前端类型注册、单元测试、Docker Compose 模板。
实现要点:
- 协议分工:主链路用 MySQL 协议(database/sql + go-sql-driver/mysql)
做 DDL / 查询 / 删除;批量更新走 Stream Load HTTP API,并启用
partial_update=true、merge_type=APPEND,自动按 1MiB 切分批次并处理
307 重定向。
- 表结构:UNIQUE KEY(id) + enable_unique_key_merge_on_write=true 以
支持 upsert/部分列更新;按维度分表(<base>_<dim>),每张表上建
HNSW ANN 索引(metric_type=cosine_distance)和 INVERTED 索引
(parser=chinese)。
- 分数语义:使用 cosine_distance_approximate,再以 1 - dist 转换为
"越大越相似",与现有 KVHybridRetrieveEngine 约定一致。
- 异步索引:ANN 索引为后台构建,ensureTable 通过轮询 SHOW INDEX 等
待索引就绪后再放行写入,避免首次检索召回为空。
- ARRAY<FLOAT> 序列化:go-sql-driver/mysql 不支持数组占位符,
embeddingLiteral 将 []float32 转成 SQL 字面量字符串再拼接。
新增文件:
- internal/application/repository/retriever/doris/{structs,schema,
query,repository,streamload,repository_test}.go
- scripts/e2e-doris.sh:E2E 验证清单
- docs/wiki/集成扩展/Doris改动与上游同步.md:fork-and-rebase 工作流
与改动清单
修改文件(接线 + 文档):
- internal/types/{retriever,tenant,vectorstore}.go:新增
DorisRetrieverEngineType、env 解析、表单 schema 与索引参数校验
- internal/container/{container,engine_factory}.go:环境变量驱动
与 VectorStore 配置驱动两条路径都支持 Doris
- internal/application/service/vectorstore{,_healthcheck}.go:连接
校验 + Ping/Version 健康检查
- docker-compose.yml:新增 doris-fe / doris-be 服务(profile=doris)
- .env.example:DORIS_* 环境变量与示例
- docs/{使用其他向量数据库,wiki/集成扩展/集成向量数据库}.md:
使用说明与索引/分数行为说明
依赖:go.mod/go.sum 新增 github.com/go-sql-driver/mysql(运行时)和
github.com/DATA-DOG/go-sqlmock(测试)。
测试:repository 层 SQL 形状、Stream Load HTTP 行为、whereBuilder
逻辑、embeddingLiteral 往返、健康检查错误路径均有单测覆盖。
272 lines
9.0 KiB
Go
272 lines
9.0 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"time"
|
||
|
||
"github.com/Tencent/WeKnora/internal/errors"
|
||
"github.com/Tencent/WeKnora/internal/logger"
|
||
"github.com/Tencent/WeKnora/internal/types"
|
||
_ "github.com/go-sql-driver/mysql" // MySQL driver for database/sql, used by Doris connection test
|
||
_ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql
|
||
"github.com/qdrant/go-client/qdrant"
|
||
"github.com/weaviate/weaviate-go-client/v5/weaviate"
|
||
"github.com/weaviate/weaviate-go-client/v5/weaviate/auth"
|
||
wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc"
|
||
)
|
||
|
||
const connectionTestTimeout = 10 * time.Second
|
||
|
||
// TestConnection tests connectivity to a vector database.
|
||
// Returns the detected server version on success (e.g., "7.10.1"), empty string if unknown.
|
||
func (s *vectorStoreService) TestConnection(
|
||
ctx context.Context,
|
||
engineType types.RetrieverEngineType,
|
||
config types.ConnectionConfig,
|
||
) (string, error) {
|
||
switch engineType {
|
||
case types.ElasticsearchRetrieverEngineType:
|
||
return testElasticsearchConnection(ctx, config)
|
||
case types.PostgresRetrieverEngineType:
|
||
return testPostgresConnection(ctx, config)
|
||
case types.QdrantRetrieverEngineType:
|
||
return testQdrantConnection(ctx, config)
|
||
case types.MilvusRetrieverEngineType:
|
||
return testMilvusConnection(ctx, config)
|
||
case types.WeaviateRetrieverEngineType:
|
||
return testWeaviateConnection(ctx, config)
|
||
case types.DorisRetrieverEngineType:
|
||
return testDorisConnection(ctx, config)
|
||
case types.SQLiteRetrieverEngineType:
|
||
// SQLite is file-based, no remote connection to test
|
||
return "", nil
|
||
default:
|
||
return "", errors.NewBadRequestError(
|
||
fmt.Sprintf("connection test not supported for engine type: %s", engineType))
|
||
}
|
||
}
|
||
|
||
func testElasticsearchConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
// Use plain HTTP GET to the root endpoint instead of the go-elasticsearch SDK.
|
||
// The v8 SDK's TypedClient performs a product check that rejects ES7 servers,
|
||
// so we use a raw HTTP request to support both v7 and v8.
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, config.Addr, nil)
|
||
if err != nil {
|
||
return "", errors.NewBadRequestError("failed to create elasticsearch request: invalid address")
|
||
}
|
||
if config.Username != "" {
|
||
req.SetBasicAuth(config.Username, config.Password)
|
||
}
|
||
|
||
client := &http.Client{Timeout: connectionTestTimeout}
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
logger.Warnf(ctx, "Elasticsearch connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to connect to elasticsearch: connection refused or authentication failed")
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
logger.Warnf(ctx, "Elasticsearch connection test returned status %d", resp.StatusCode)
|
||
return "", errors.NewBadRequestError("failed to connect to elasticsearch: authentication failed or server error")
|
||
}
|
||
|
||
// Parse version from response: {"version": {"number": "7.10.1"}, ...}
|
||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||
if err != nil {
|
||
return "", nil // connected but version unknown
|
||
}
|
||
|
||
var esInfo struct {
|
||
Version struct {
|
||
Number string `json:"number"`
|
||
} `json:"version"`
|
||
}
|
||
if err := json.Unmarshal(body, &esInfo); err != nil {
|
||
return "", nil // connected but version unparseable
|
||
}
|
||
|
||
return esInfo.Version.Number, nil
|
||
}
|
||
|
||
func testPostgresConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||
defer cancel()
|
||
|
||
if config.UseDefaultConnection {
|
||
// Using the default app DB connection — always reachable if the app is running.
|
||
// Cannot query version without a DB handle; return empty.
|
||
return "", nil
|
||
}
|
||
|
||
db, err := sql.Open("pgx", config.Addr)
|
||
if err != nil {
|
||
return "", errors.NewBadRequestError("failed to create postgres connection: invalid configuration")
|
||
}
|
||
defer db.Close()
|
||
|
||
if err := db.PingContext(testCtx); err != nil {
|
||
logger.Warnf(ctx, "Postgres connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to connect to postgres: connection refused or authentication failed")
|
||
}
|
||
|
||
// Detect version
|
||
var version string
|
||
if err := db.QueryRowContext(testCtx, "SHOW server_version").Scan(&version); err != nil {
|
||
logger.Warnf(ctx, "Postgres version detection failed: %v", err)
|
||
return "", nil // connected but version unknown
|
||
}
|
||
|
||
return version, nil
|
||
}
|
||
|
||
func testQdrantConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||
defer cancel()
|
||
|
||
port := config.Port
|
||
if port == 0 {
|
||
port = 6334
|
||
}
|
||
|
||
client, err := qdrant.NewClient(&qdrant.Config{
|
||
Host: config.Host,
|
||
Port: port,
|
||
APIKey: config.APIKey,
|
||
UseTLS: config.UseTLS,
|
||
})
|
||
if err != nil {
|
||
return "", errors.NewBadRequestError("failed to create qdrant client: invalid configuration")
|
||
}
|
||
defer client.Close()
|
||
|
||
result, err := client.HealthCheck(testCtx)
|
||
if err != nil {
|
||
logger.Warnf(ctx, "Qdrant connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to connect to qdrant: connection refused or authentication failed")
|
||
}
|
||
|
||
return result.GetVersion(), nil
|
||
}
|
||
|
||
func testMilvusConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
// Use TCP dial instead of the Milvus SDK to avoid protobuf namespace conflict
|
||
// between milvus-proto and qdrant-client (both register "common.proto").
|
||
// A TCP dial is sufficient for connectivity verification; the Milvus SDK client
|
||
// creation in container.go (PR 3) will validate full gRPC connectivity.
|
||
// Version detection is not possible with TCP dial alone.
|
||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||
defer cancel()
|
||
|
||
addr := config.Addr
|
||
if addr == "" {
|
||
addr = "localhost:19530"
|
||
}
|
||
|
||
conn, err := (&net.Dialer{}).DialContext(testCtx, "tcp", addr)
|
||
if err != nil {
|
||
logger.Warnf(ctx, "Milvus connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to connect to milvus: connection refused or server unreachable")
|
||
}
|
||
defer conn.Close()
|
||
|
||
return "", nil
|
||
}
|
||
|
||
func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||
defer cancel()
|
||
|
||
host := config.Host
|
||
if host == "" {
|
||
host = "weaviate:8080"
|
||
}
|
||
grpcAddress := config.GrpcAddress
|
||
if grpcAddress == "" {
|
||
grpcAddress = "weaviate:50051"
|
||
}
|
||
scheme := config.Scheme
|
||
if scheme == "" {
|
||
scheme = "http"
|
||
}
|
||
|
||
weaviateCfg := weaviate.Config{
|
||
Host: host,
|
||
GrpcConfig: &wgrpc.Config{
|
||
Host: grpcAddress,
|
||
},
|
||
Scheme: scheme,
|
||
}
|
||
if config.APIKey != "" {
|
||
weaviateCfg.AuthConfig = auth.ApiKey{Value: config.APIKey}
|
||
}
|
||
|
||
// Weaviate Go client v5 does not expose a Close() method;
|
||
// it uses HTTP + gRPC transports that are managed internally.
|
||
client, err := weaviate.NewClient(weaviateCfg)
|
||
if err != nil {
|
||
logger.Warnf(ctx, "Weaviate connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to create weaviate client: invalid configuration")
|
||
}
|
||
|
||
isReady, err := client.Misc().ReadyChecker().Do(testCtx)
|
||
if err != nil || !isReady {
|
||
logger.Warnf(ctx, "Weaviate connection test failed: ready=%v, err=%v", isReady, err)
|
||
return "", errors.NewBadRequestError("failed to connect to weaviate: server not ready or authentication failed")
|
||
}
|
||
|
||
// Detect version via /v1/meta
|
||
meta, err := client.Misc().MetaGetter().Do(testCtx)
|
||
if err != nil || meta == nil {
|
||
return "", nil // connected but version unknown
|
||
}
|
||
|
||
return meta.Version, nil
|
||
}
|
||
|
||
// testDorisConnection 通过 MySQL 协议(database/sql + go-sql-driver)
|
||
// Ping Doris FE 并查询 @@version。
|
||
//
|
||
// Doris 4.1 的 @@version 形如 "5.7.99 Doris-4.1.0"——前半段是兼容性表达式,
|
||
// 后半段才是真正的 Doris 版本号。这里直接返回原样字符串,由调用方按需展示。
|
||
func testDorisConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||
defer cancel()
|
||
|
||
if config.Addr == "" {
|
||
return "", errors.NewBadRequestError("failed to create doris connection: addr is required")
|
||
}
|
||
|
||
// Database 不强制要求;Ping 时无明确库则用 information_schema(任何 MySQL 兼容服务都有)。
|
||
database := config.Database
|
||
if database == "" {
|
||
database = "information_schema"
|
||
}
|
||
|
||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?timeout=5s",
|
||
config.Username, config.Password, config.Addr, database)
|
||
db, err := sql.Open("mysql", dsn)
|
||
if err != nil {
|
||
return "", errors.NewBadRequestError("failed to create doris connection: invalid configuration")
|
||
}
|
||
defer db.Close()
|
||
|
||
if err := db.PingContext(testCtx); err != nil {
|
||
logger.Warnf(ctx, "Doris connection test failed: %v", err)
|
||
return "", errors.NewBadRequestError("failed to connect to doris: connection refused or authentication failed")
|
||
}
|
||
|
||
var version string
|
||
if err := db.QueryRowContext(testCtx, "SELECT @@version").Scan(&version); err != nil {
|
||
logger.Warnf(ctx, "Doris version detection failed: %v", err)
|
||
return "", nil
|
||
}
|
||
return version, nil
|
||
}
|