feat(retriever): 接入 Apache Doris 4.1 作为向量数据库
为 RetrieveEngine 体系新增 Doris 后端,与现有 Qdrant/Milvus/Weaviate
等保持完整能力对齐:向量检索、关键词检索、健康检查、环境变量与多实例
DB 配置、前端类型注册、单元测试、Docker Compose 模板。
实现要点:
- 协议分工:主链路用 MySQL 协议(database/sql + go-sql-driver/mysql)
做 DDL / 查询 / 删除;批量更新走 Stream Load HTTP API,并启用
partial_update=true、merge_type=APPEND,自动按 1MiB 切分批次并处理
307 重定向。
- 表结构:UNIQUE KEY(id) + enable_unique_key_merge_on_write=true 以
支持 upsert/部分列更新;按维度分表(<base>_<dim>),每张表上建
HNSW ANN 索引(metric_type=cosine_distance)和 INVERTED 索引
(parser=chinese)。
- 分数语义:使用 cosine_distance_approximate,再以 1 - dist 转换为
"越大越相似",与现有 KVHybridRetrieveEngine 约定一致。
- 异步索引:ANN 索引为后台构建,ensureTable 通过轮询 SHOW INDEX 等
待索引就绪后再放行写入,避免首次检索召回为空。
- ARRAY<FLOAT> 序列化:go-sql-driver/mysql 不支持数组占位符,
embeddingLiteral 将 []float32 转成 SQL 字面量字符串再拼接。
新增文件:
- internal/application/repository/retriever/doris/{structs,schema,
query,repository,streamload,repository_test}.go
- scripts/e2e-doris.sh:E2E 验证清单
- docs/wiki/集成扩展/Doris改动与上游同步.md:fork-and-rebase 工作流
与改动清单
修改文件(接线 + 文档):
- internal/types/{retriever,tenant,vectorstore}.go:新增
DorisRetrieverEngineType、env 解析、表单 schema 与索引参数校验
- internal/container/{container,engine_factory}.go:环境变量驱动
与 VectorStore 配置驱动两条路径都支持 Doris
- internal/application/service/vectorstore{,_healthcheck}.go:连接
校验 + Ping/Version 健康检查
- docker-compose.yml:新增 doris-fe / doris-be 服务(profile=doris)
- .env.example:DORIS_* 环境变量与示例
- docs/{使用其他向量数据库,wiki/集成扩展/集成向量数据库}.md:
使用说明与索引/分数行为说明
依赖:go.mod/go.sum 新增 github.com/go-sql-driver/mysql(运行时)和
github.com/DATA-DOG/go-sqlmock(测试)。
测试:repository 层 SQL 形状、Stream Load HTTP 行为、whereBuilder
逻辑、embeddingLiteral 往返、健康检查错误路径均有单测覆盖。
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// 字段名常量。Doris 是 SQL 库,字段名要在 SELECT/WHERE/INSERT 多处复用,
|
||||
// 用常量统一防止笔误。
|
||||
const (
|
||||
fieldID = "id"
|
||||
fieldContent = "content"
|
||||
fieldSourceID = "source_id"
|
||||
fieldSourceType = "source_type"
|
||||
fieldChunkID = "chunk_id"
|
||||
fieldKnowledgeID = "knowledge_id"
|
||||
fieldKnowledgeBaseID = "knowledge_base_id"
|
||||
fieldTagID = "tag_id"
|
||||
fieldIsEnabled = "is_enabled"
|
||||
fieldEmbedding = "embedding"
|
||||
)
|
||||
|
||||
// columns 是 INSERT / SELECT 时使用的标准列序。
|
||||
var columns = []string{
|
||||
fieldID, fieldContent, fieldSourceID, fieldSourceType,
|
||||
fieldChunkID, fieldKnowledgeID, fieldKnowledgeBaseID, fieldTagID,
|
||||
fieldIsEnabled, fieldEmbedding,
|
||||
}
|
||||
|
||||
// columnsForRetrieve 是 Retrieve 时 SELECT 的列序,
|
||||
// 不包含 embedding(向量本身查询结果中无需返回,省带宽)。
|
||||
var columnsForRetrieve = []string{
|
||||
fieldID, fieldContent, fieldSourceID, fieldSourceType,
|
||||
fieldChunkID, fieldKnowledgeID, fieldKnowledgeBaseID, fieldTagID,
|
||||
fieldIsEnabled,
|
||||
}
|
||||
|
||||
// columnsForCopy 是 CopyIndices 中分页 SELECT 时使用的列序,
|
||||
// 比 columnsForRetrieve 多 embedding,因为复制目的是搬运向量本身。
|
||||
var columnsForCopy = []string{
|
||||
fieldID, fieldContent, fieldSourceID, fieldSourceType,
|
||||
fieldChunkID, fieldKnowledgeID, fieldKnowledgeBaseID, fieldTagID,
|
||||
fieldIsEnabled, fieldEmbedding,
|
||||
}
|
||||
|
||||
// whereCond 表示一个 WHERE 子条件:clause 是参数化 SQL 片段(带 ? 占位),
|
||||
// args 是对应顺序的参数值。所有用户输入字段(IDs)必须通过 args 传入,
|
||||
// 严禁拼到 clause 字符串里。
|
||||
type whereCond struct {
|
||||
clause string
|
||||
args []any
|
||||
}
|
||||
|
||||
// whereBuilder 用于把 RetrieveParams 中的过滤条件翻译成 SQL WHERE 子句。
|
||||
//
|
||||
// 每个 add* 方法对应一种 IN / NOT IN / = 算子;最终 build() 用 AND 拼接。
|
||||
type whereBuilder struct {
|
||||
conds []whereCond
|
||||
}
|
||||
|
||||
// addEqual 追加一个 field = ? 条件。
|
||||
func (w *whereBuilder) addEqual(field string, value any) {
|
||||
w.conds = append(w.conds, whereCond{
|
||||
clause: field + " = ?",
|
||||
args: []any{value},
|
||||
})
|
||||
}
|
||||
|
||||
// addIn 追加一个 field IN (?, ?, ...) 条件。values 为空时不追加任何东西。
|
||||
func (w *whereBuilder) addIn(field string, values []string) {
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
placeholders := make([]string, len(values))
|
||||
args := make([]any, len(values))
|
||||
for i, v := range values {
|
||||
placeholders[i] = "?"
|
||||
args[i] = v
|
||||
}
|
||||
w.conds = append(w.conds, whereCond{
|
||||
clause: field + " IN (" + strings.Join(placeholders, ", ") + ")",
|
||||
args: args,
|
||||
})
|
||||
}
|
||||
|
||||
// addNotIn 追加一个 field NOT IN (?, ?, ...) 条件。
|
||||
func (w *whereBuilder) addNotIn(field string, values []string) {
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
placeholders := make([]string, len(values))
|
||||
args := make([]any, len(values))
|
||||
for i, v := range values {
|
||||
placeholders[i] = "?"
|
||||
args[i] = v
|
||||
}
|
||||
w.conds = append(w.conds, whereCond{
|
||||
clause: field + " NOT IN (" + strings.Join(placeholders, ", ") + ")",
|
||||
args: args,
|
||||
})
|
||||
}
|
||||
|
||||
// build 返回 WHERE 子句(不含 "WHERE " 前缀)和参数数组。
|
||||
// 没有任何条件时返回 ("1 = 1", nil),方便调用方无脑拼接。
|
||||
func (w *whereBuilder) build() (string, []any) {
|
||||
if len(w.conds) == 0 {
|
||||
return "1 = 1", nil
|
||||
}
|
||||
parts := make([]string, len(w.conds))
|
||||
var args []any
|
||||
for i, c := range w.conds {
|
||||
parts[i] = c.clause
|
||||
args = append(args, c.args...)
|
||||
}
|
||||
return strings.Join(parts, " AND "), args
|
||||
}
|
||||
|
||||
// buildBaseFilter 将 RetrieveParams 中的过滤条件翻译为 whereBuilder。
|
||||
// 默认追加 is_enabled = TRUE,与 Qdrant/Milvus/Weaviate 保持一致:
|
||||
// 关闭的 chunk 不参与检索。
|
||||
func buildBaseFilter(params types.RetrieveParams) *whereBuilder {
|
||||
w := &whereBuilder{}
|
||||
w.addEqual(fieldIsEnabled, true)
|
||||
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
w.addIn(fieldKnowledgeBaseID, params.KnowledgeBaseIDs)
|
||||
}
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
w.addIn(fieldKnowledgeID, params.KnowledgeIDs)
|
||||
}
|
||||
if len(params.TagIDs) > 0 {
|
||||
w.addIn(fieldTagID, params.TagIDs)
|
||||
}
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
w.addNotIn(fieldKnowledgeID, params.ExcludeKnowledgeIDs)
|
||||
}
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
w.addNotIn(fieldChunkID, params.ExcludeChunkIDs)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// parseEmbeddingLiteral 解析 Doris ARRAY<FLOAT> 通过 MySQL 协议返回的
|
||||
// 字面量字符串(形如 "[1,2,3]")为 []float32。
|
||||
//
|
||||
// CopyIndices 路径需要从源行读出向量本身再写回目标行;此处的解析容错优先:
|
||||
// 不带 [] 也接受、空数组返回 nil。
|
||||
func parseEmbeddingLiteral(raw []byte) ([]float32, error) {
|
||||
s := strings.TrimSpace(string(raw))
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
s = strings.TrimPrefix(s, "[")
|
||||
s = strings.TrimSuffix(s, "]")
|
||||
if s == "" {
|
||||
return nil, nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := make([]float32, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
f, err := strconv.ParseFloat(p, 32)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, float32(f))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// embeddingLiteral 把 []float32 转为 Doris ARRAY<FLOAT> 字面量字符串:
|
||||
// "[1.23,4.56,...]"。
|
||||
//
|
||||
// 为何不用占位符:go-sql-driver/mysql 不支持 ARRAY 类型的参数绑定,
|
||||
// Doris 端也只接受字面量形式。这里用 strconv.FormatFloat('g' + bitSize=32)
|
||||
// 而不是 fmt.Sprintf("%f", v),原因有二:
|
||||
// 1. fmt 在某些 locale 下会用千分位分隔符,破坏 SQL 语法;
|
||||
// 2. 'g' 比 'f' 短且不会丢精度。
|
||||
//
|
||||
// 注入风险:[]float32 元素是嵌入模型输出的有限位数浮点数,序列化后只可能
|
||||
// 包含 [0-9eE+-.\s] 字符,不会逃逸出字面量上下文。
|
||||
func embeddingLiteral(vec []float32) string {
|
||||
if len(vec) == 0 {
|
||||
return "[]"
|
||||
}
|
||||
var sb strings.Builder
|
||||
sb.Grow(len(vec) * 12)
|
||||
sb.WriteByte('[')
|
||||
for i, v := range vec {
|
||||
if i > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
sb.WriteString(strconv.FormatFloat(float64(v), 'g', -1, 32))
|
||||
}
|
||||
sb.WriteByte(']')
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,584 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTableBaseName = "weknora_embeddings"
|
||||
envDorisTablePrefix = "DORIS_TABLE_PREFIX"
|
||||
)
|
||||
|
||||
// NewDorisRetrieveEngineRepository 创建 Doris 检索引擎仓储。
|
||||
//
|
||||
// 参数:
|
||||
// - db:MySQL 协议的 *sql.DB 实例。调用方负责 SetMaxOpenConns 等参数。
|
||||
// - feHTTPBase:Stream Load 用的 FE HTTP 基地址(含 scheme),例如 "http://doris-fe:8030"。
|
||||
// - username/password:MySQL 与 Stream Load 共用的凭据。
|
||||
// - database:目标数据库名(既用于 MySQL DSN,也用于 Stream Load URL 路径)。
|
||||
// - indexCfg:可空。为 nil 时退化为环境变量 + 默认值(env 路径)。
|
||||
func NewDorisRetrieveEngineRepository(
|
||||
db *sql.DB,
|
||||
feHTTPBase, username, password, database string,
|
||||
indexCfg *types.IndexConfig,
|
||||
) interfaces.RetrieveEngineRepository {
|
||||
log := logger.GetLogger(context.Background())
|
||||
log.Info("[Doris] Initializing Doris retriever engine repository")
|
||||
|
||||
tableBaseName := types.ResolveCollectionName(indexCfg, envDorisTablePrefix, defaultTableBaseName)
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
httpClient: &http.Client{},
|
||||
feHTTPBase: strings.TrimRight(feHTTPBase, "/"),
|
||||
username: username,
|
||||
password: password,
|
||||
database: database,
|
||||
tableBaseName: tableBaseName,
|
||||
bucketsNum: indexCfg.GetBucketsNum(0),
|
||||
replicationNum: indexCfg.GetReplicationNum(0),
|
||||
}
|
||||
log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s",
|
||||
database, tableBaseName, repo.feHTTPBase)
|
||||
return repo
|
||||
}
|
||||
|
||||
func (r *dorisRepository) EngineType() types.RetrieverEngineType {
|
||||
return types.DorisRetrieverEngineType
|
||||
}
|
||||
|
||||
func (r *dorisRepository) Support() []types.RetrieverType {
|
||||
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
||||
}
|
||||
|
||||
// EstimateStorageSize 估算给定 IndexInfo 列表的存储字节数。
|
||||
//
|
||||
// 参考 Qdrant 的算法:payload 字段长度 + 向量字节 + HNSW 邻居 + 元数据。
|
||||
func (r *dorisRepository) EstimateStorageSize(_ context.Context,
|
||||
indexInfoList []*types.IndexInfo, params map[string]any,
|
||||
) int64 {
|
||||
var total int64
|
||||
for _, info := range indexInfoList {
|
||||
emb := toDorisVectorEmbedding(info, params)
|
||||
total += calculateStorageSize(emb)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Save 写入单条记录到对应维度的表。
|
||||
func (r *dorisRepository) Save(ctx context.Context,
|
||||
info *types.IndexInfo, additionalParams map[string]any,
|
||||
) error {
|
||||
emb := toDorisVectorEmbedding(info, additionalParams)
|
||||
if len(emb.Embedding) == 0 {
|
||||
return fmt.Errorf("empty embedding vector for chunk ID: %s", info.ChunkID)
|
||||
}
|
||||
return r.BatchSave(ctx, []*types.IndexInfo{info}, additionalParams)
|
||||
}
|
||||
|
||||
// BatchSave 把同一批 IndexInfo 按维度分组,对每个维度构造一条
|
||||
// INSERT INTO ... VALUES (...), (...) 语句。UNIQUE KEY 表会自动按 id upsert。
|
||||
func (r *dorisRepository) BatchSave(ctx context.Context,
|
||||
indexInfoList []*types.IndexInfo, additionalParams map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(indexInfoList) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
groups := make(map[int][]*DorisVectorEmbedding)
|
||||
for _, info := range indexInfoList {
|
||||
emb := toDorisVectorEmbedding(info, additionalParams)
|
||||
if len(emb.Embedding) == 0 {
|
||||
log.Warnf("[Doris] Skipping empty embedding for chunk %s", info.ChunkID)
|
||||
continue
|
||||
}
|
||||
// 给一个稳定的主键。SourceID 是上层最有意义的"行身份",
|
||||
// 但同 chunk 多 question 的场景下 SourceID 已经唯一,所以直接用它。
|
||||
if emb.ID == "" {
|
||||
emb.ID = emb.SourceID
|
||||
}
|
||||
if emb.ID == "" {
|
||||
emb.ID = uuid.New().String()
|
||||
}
|
||||
dim := len(emb.Embedding)
|
||||
groups[dim] = append(groups[dim], emb)
|
||||
}
|
||||
|
||||
for dim, rows := range groups {
|
||||
if err := r.ensureTable(ctx, dim); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.insertRows(ctx, r.getTableName(dim), rows); err != nil {
|
||||
return fmt.Errorf("batch save dim=%d: %w", dim, err)
|
||||
}
|
||||
log.Infof("[Doris] Saved %d rows to %s", len(rows), r.getTableName(dim))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertRows 按列序拼一条多 VALUES 的 INSERT。embedding 列由于
|
||||
// go-sql-driver/mysql 不支持 ARRAY 占位符,必须以字面量形式拼到 SQL 文本中。
|
||||
func (r *dorisRepository) insertRows(ctx context.Context,
|
||||
table string, rows []*DorisVectorEmbedding,
|
||||
) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 9 个普通占位符 + 1 个 embedding 字面量。
|
||||
const perRowPlaceholders = "(?, ?, ?, ?, ?, ?, ?, ?, ?, %s)"
|
||||
|
||||
parts := make([]string, len(rows))
|
||||
args := make([]any, 0, len(rows)*9)
|
||||
for i, e := range rows {
|
||||
parts[i] = fmt.Sprintf(perRowPlaceholders, embeddingLiteral(e.Embedding))
|
||||
args = append(args,
|
||||
e.ID, e.Content, e.SourceID, e.SourceType,
|
||||
e.ChunkID, e.KnowledgeID, e.KnowledgeBaseID, e.TagID,
|
||||
e.IsEnabled,
|
||||
)
|
||||
}
|
||||
|
||||
stmt := fmt.Sprintf("INSERT INTO `%s` (%s) VALUES %s",
|
||||
table,
|
||||
strings.Join(columns, ", "),
|
||||
strings.Join(parts, ", "),
|
||||
)
|
||||
_, err := r.db.ExecContext(ctx, stmt, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList 用 chunk_id 列删除。dimension 用于定位具体表。
|
||||
func (r *dorisRepository) DeleteByChunkIDList(ctx context.Context,
|
||||
chunkIDList []string, dimension int, _ string,
|
||||
) error {
|
||||
return r.deleteByField(ctx, fieldChunkID, chunkIDList, dimension)
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList 用 knowledge_id 列删除。
|
||||
func (r *dorisRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int, _ string,
|
||||
) error {
|
||||
return r.deleteByField(ctx, fieldKnowledgeID, knowledgeIDList, dimension)
|
||||
}
|
||||
|
||||
// DeleteBySourceIDList 用 source_id 列删除。
|
||||
func (r *dorisRepository) DeleteBySourceIDList(ctx context.Context,
|
||||
sourceIDList []string, dimension int, _ string,
|
||||
) error {
|
||||
return r.deleteByField(ctx, fieldSourceID, sourceIDList, dimension)
|
||||
}
|
||||
|
||||
// deleteByField 是三个 Delete* 方法的统一实现:
|
||||
// DELETE FROM <table> WHERE <field> IN (?, ?, ...)。
|
||||
func (r *dorisRepository) deleteByField(ctx context.Context,
|
||||
field string, ids []string, dimension int,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
table := r.getTableName(dimension)
|
||||
placeholders := make([]string, len(ids))
|
||||
args := make([]any, len(ids))
|
||||
for i, v := range ids {
|
||||
placeholders[i] = "?"
|
||||
args[i] = v
|
||||
}
|
||||
stmt := fmt.Sprintf("DELETE FROM `%s` WHERE %s IN (%s)",
|
||||
table, field, strings.Join(placeholders, ", "))
|
||||
|
||||
if _, err := r.db.ExecContext(ctx, stmt, args...); err != nil {
|
||||
log.Errorf("[Doris] Delete by %s failed: %v", field, err)
|
||||
return fmt.Errorf("delete by %s: %w", field, err)
|
||||
}
|
||||
log.Infof("[Doris] Deleted %d rows from %s by %s", len(ids), table, field)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retrieve 根据 RetrieverType 分发到向量检索或关键词检索。
|
||||
func (r *dorisRepository) Retrieve(ctx context.Context,
|
||||
params types.RetrieveParams,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
switch params.RetrieverType {
|
||||
case types.VectorRetrieverType:
|
||||
return r.VectorRetrieve(ctx, params)
|
||||
case types.KeywordsRetrieverType:
|
||||
return r.KeywordsRetrieve(ctx, params)
|
||||
}
|
||||
return nil, fmt.Errorf("invalid retriever type: %v", params.RetrieverType)
|
||||
}
|
||||
|
||||
// VectorRetrieve 调用 cosine_distance_approximate 做 ANN 搜索,
|
||||
// score = 1 - distance 与 Qdrant cosine 相似度方向一致:值越大越相似。
|
||||
func (r *dorisRepository) VectorRetrieve(ctx context.Context,
|
||||
params types.RetrieveParams,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
dim := len(params.Embedding)
|
||||
table := r.getTableName(dim)
|
||||
|
||||
exists, err := r.tableExists(ctx, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("check table %s: %w", table, err)
|
||||
}
|
||||
if !exists {
|
||||
log.Warnf("[Doris] Table %s does not exist, returning empty results", table)
|
||||
return buildRetrieveResult(nil, types.VectorRetrieverType), nil
|
||||
}
|
||||
|
||||
wb := buildBaseFilter(params)
|
||||
whereClause, whereArgs := wb.build()
|
||||
|
||||
// embedding 必须用字面量,threshold/topK 用占位符。
|
||||
// 使用 HAVING 是因为 score 是 SELECT 列别名,WHERE 阶段还看不到。
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s, (1 - cosine_distance_approximate(`%s`, %s)) AS score "+
|
||||
"FROM `%s` WHERE %s "+
|
||||
"HAVING score >= ? "+
|
||||
"ORDER BY score DESC LIMIT ?",
|
||||
strings.Join(columnsForRetrieve, ", "),
|
||||
fieldEmbedding,
|
||||
embeddingLiteral(params.Embedding),
|
||||
table,
|
||||
whereClause,
|
||||
)
|
||||
args := append(whereArgs, params.Threshold, params.TopK)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("vector retrieve %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
results, err := scanRetrieveRows(rows, types.MatchTypeEmbedding)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Infof("[Doris] Vector retrieval found %d results in %s", len(results), table)
|
||||
return buildRetrieveResult(results, types.VectorRetrieverType), nil
|
||||
}
|
||||
|
||||
// KeywordsRetrieve 用 Doris 倒排索引 + MATCH_ANY 做关键词匹配。
|
||||
//
|
||||
// 不需要 jieba 客户端分词:CREATE TABLE 时 idx_content 已经声明了 chinese parser。
|
||||
// 不同维度的表跨表合并取 topK,与 Milvus/Weaviate 现状一致。
|
||||
func (r *dorisRepository) KeywordsRetrieve(ctx context.Context,
|
||||
params types.RetrieveParams,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
query := strings.TrimSpace(params.Query)
|
||||
if query == "" {
|
||||
return buildRetrieveResult(nil, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
tables, err := r.listEmbeddingTables(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tables: %w", err)
|
||||
}
|
||||
if len(tables) == 0 {
|
||||
return buildRetrieveResult(nil, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
wb := buildBaseFilter(params)
|
||||
whereClause, whereArgs := wb.build()
|
||||
|
||||
var all []*types.IndexWithScore
|
||||
for _, table := range tables {
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s FROM `%s` WHERE %s AND %s MATCH_ANY ? LIMIT ?",
|
||||
strings.Join(columnsForRetrieve, ", "),
|
||||
table, whereClause, fieldContent,
|
||||
)
|
||||
args := append(append([]any{}, whereArgs...), query, params.TopK)
|
||||
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
log.Warnf("[Doris] Keyword retrieve in %s failed: %v", table, err)
|
||||
continue
|
||||
}
|
||||
// score 在 KeywordsRetrieve 中固定 1.0,与 Qdrant 行为一致。
|
||||
batch, scanErr := scanRetrieveRows(rows, types.MatchTypeKeywords)
|
||||
_ = rows.Close()
|
||||
if scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
all = append(all, batch...)
|
||||
}
|
||||
if len(all) > params.TopK {
|
||||
all = all[:params.TopK]
|
||||
}
|
||||
log.Infof("[Doris] Keywords retrieval found %d results across %d tables", len(all), len(tables))
|
||||
return buildRetrieveResult(all, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
// CopyIndices 把源知识库的 chunk 复制到目标知识库,避免重新生成 embedding。
|
||||
//
|
||||
// 与 Qdrant 的实现完全镜像:
|
||||
// - 分页扫描源表
|
||||
// - 按 sourceToTargetChunkIDMap 把 chunk_id 翻译过去
|
||||
// - 处理 source_id 翻译规则(普通 chunk / 生成型问题 / 其他)
|
||||
// - 把目标行写回同一个表
|
||||
func (r *dorisRepository) CopyIndices(ctx context.Context,
|
||||
sourceKnowledgeBaseID string,
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
_ string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(sourceToTargetChunkIDMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := r.ensureTable(ctx, dimension); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
table := r.getTableName(dimension)
|
||||
const pageSize = 64
|
||||
offset := 0
|
||||
totalCopied := 0
|
||||
|
||||
for {
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s FROM `%s` WHERE %s = ? ORDER BY %s LIMIT ? OFFSET ?",
|
||||
strings.Join(columnsForCopy, ", "),
|
||||
table, fieldKnowledgeBaseID, fieldID,
|
||||
)
|
||||
rows, err := r.db.QueryContext(ctx, stmt, sourceKnowledgeBaseID, pageSize, offset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copy indices scan: %w", err)
|
||||
}
|
||||
batch, err := scanCopyRows(rows)
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
var targets []*DorisVectorEmbedding
|
||||
for _, src := range batch {
|
||||
targetChunkID, ok := sourceToTargetChunkIDMap[src.ChunkID]
|
||||
if !ok {
|
||||
log.Warnf("[Doris] Source chunk %s not in target mapping", src.ChunkID)
|
||||
continue
|
||||
}
|
||||
targetKnowledgeID, ok := sourceToTargetKBIDMap[src.KnowledgeID]
|
||||
if !ok {
|
||||
log.Warnf("[Doris] Source knowledge %s not in target mapping", src.KnowledgeID)
|
||||
continue
|
||||
}
|
||||
|
||||
targetSourceID := translateSourceID(src.SourceID, src.ChunkID, targetChunkID)
|
||||
targets = append(targets, &DorisVectorEmbedding{
|
||||
ID: uuid.New().String(),
|
||||
Content: src.Content,
|
||||
SourceID: targetSourceID,
|
||||
SourceType: src.SourceType,
|
||||
ChunkID: targetChunkID,
|
||||
KnowledgeID: targetKnowledgeID,
|
||||
KnowledgeBaseID: targetKnowledgeBaseID,
|
||||
TagID: src.TagID,
|
||||
IsEnabled: src.IsEnabled,
|
||||
Embedding: src.Embedding,
|
||||
})
|
||||
}
|
||||
|
||||
if len(targets) > 0 {
|
||||
if err := r.insertRows(ctx, table, targets); err != nil {
|
||||
return fmt.Errorf("copy indices insert: %w", err)
|
||||
}
|
||||
totalCopied += len(targets)
|
||||
}
|
||||
|
||||
if len(batch) < pageSize {
|
||||
break
|
||||
}
|
||||
offset += pageSize
|
||||
}
|
||||
log.Infof("[Doris] CopyIndices done, dim=%d, copied=%d", dimension, totalCopied)
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus / BatchUpdateChunkTagID 实际实现位于 streamload.go,
|
||||
// 通过 Stream Load partial update 协议执行高性能字段级更新。
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 私有辅助
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// toDorisVectorEmbedding 把 IndexInfo + 上层传入的 embedding 映射 转换为
|
||||
// Doris 行模型。Embedding 通过 additionalParams[fieldEmbedding] 中的
|
||||
// map[string][]float32 按 SourceID 取出,与 Qdrant/Milvus 完全一致。
|
||||
func toDorisVectorEmbedding(info *types.IndexInfo, additionalParams map[string]any) *DorisVectorEmbedding {
|
||||
emb := &DorisVectorEmbedding{
|
||||
ID: info.ID,
|
||||
Content: info.Content,
|
||||
SourceID: info.SourceID,
|
||||
SourceType: int(info.SourceType),
|
||||
ChunkID: info.ChunkID,
|
||||
KnowledgeID: info.KnowledgeID,
|
||||
KnowledgeBaseID: info.KnowledgeBaseID,
|
||||
TagID: info.TagID,
|
||||
IsEnabled: info.IsEnabled,
|
||||
}
|
||||
if additionalParams != nil {
|
||||
if v, ok := additionalParams[fieldEmbedding]; ok {
|
||||
if m, ok := v.(map[string][]float32); ok {
|
||||
emb.Embedding = m[info.SourceID]
|
||||
}
|
||||
}
|
||||
}
|
||||
return emb
|
||||
}
|
||||
|
||||
// translateSourceID 把源 SourceID 翻译到目标 SourceID,与 Qdrant 实现完全镜像:
|
||||
// - 普通 chunk:SourceID == ChunkID -> 使用 targetChunkID
|
||||
// - 生成型问题:SourceID == "<chunkID>-<questionID>" -> "<targetChunkID>-<questionID>"
|
||||
// - 其他场景:生成新的 UUID(保持唯一性)
|
||||
func translateSourceID(originalSourceID, sourceChunkID, targetChunkID string) string {
|
||||
switch {
|
||||
case originalSourceID == sourceChunkID:
|
||||
return targetChunkID
|
||||
case strings.HasPrefix(originalSourceID, sourceChunkID+"-"):
|
||||
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
|
||||
return fmt.Sprintf("%s-%s", targetChunkID, questionID)
|
||||
default:
|
||||
return uuid.New().String()
|
||||
}
|
||||
}
|
||||
|
||||
// scanRetrieveRows 把 Retrieve 阶段的 rows 反序列化为 IndexWithScore 列表。
|
||||
//
|
||||
// 分两路:
|
||||
// - 列数 == columnsForRetrieve+1:第 N+1 列是 score(向量检索路径)
|
||||
// - 列数 == columnsForRetrieve:score 统一赋 1.0(关键词检索路径)
|
||||
func scanRetrieveRows(rows *sql.Rows, matchType types.MatchType) ([]*types.IndexWithScore, error) {
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
withScore := len(cols) == len(columnsForRetrieve)+1
|
||||
|
||||
var out []*types.IndexWithScore
|
||||
for rows.Next() {
|
||||
var (
|
||||
id, content, sourceID, chunkID string
|
||||
knowledgeID, knowledgeBaseID, tagID string
|
||||
sourceType int
|
||||
isEnabled bool
|
||||
score float64
|
||||
err error
|
||||
)
|
||||
if withScore {
|
||||
err = rows.Scan(&id, &content, &sourceID, &sourceType,
|
||||
&chunkID, &knowledgeID, &knowledgeBaseID, &tagID, &isEnabled, &score)
|
||||
} else {
|
||||
err = rows.Scan(&id, &content, &sourceID, &sourceType,
|
||||
&chunkID, &knowledgeID, &knowledgeBaseID, &tagID, &isEnabled)
|
||||
score = 1.0
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan row: %w", err)
|
||||
}
|
||||
out = append(out, &types.IndexWithScore{
|
||||
ID: id,
|
||||
Content: content,
|
||||
SourceID: sourceID,
|
||||
SourceType: types.SourceType(sourceType),
|
||||
ChunkID: chunkID,
|
||||
KnowledgeID: knowledgeID,
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
TagID: tagID,
|
||||
Score: score,
|
||||
MatchType: matchType,
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// scanCopyRows 反序列化 CopyIndices 的分页查询结果。
|
||||
//
|
||||
// 与 scanRetrieveRows 不同,这里需要 embedding 字段(去复制原始向量)。
|
||||
// Doris 的 ARRAY<FLOAT> 通过 mysql 协议返回的是字符串字面量 "[1,2,3]"。
|
||||
func scanCopyRows(rows *sql.Rows) ([]*DorisVectorEmbedding, error) {
|
||||
var out []*DorisVectorEmbedding
|
||||
for rows.Next() {
|
||||
var (
|
||||
id, content, sourceID, chunkID string
|
||||
knowledgeID, knowledgeBaseID, tagID string
|
||||
sourceType int
|
||||
isEnabled bool
|
||||
embeddingRaw sql.RawBytes
|
||||
)
|
||||
if err := rows.Scan(&id, &content, &sourceID, &sourceType,
|
||||
&chunkID, &knowledgeID, &knowledgeBaseID, &tagID, &isEnabled, &embeddingRaw); err != nil {
|
||||
return nil, fmt.Errorf("scan copy row: %w", err)
|
||||
}
|
||||
vec, err := parseEmbeddingLiteral(embeddingRaw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse embedding: %w", err)
|
||||
}
|
||||
out = append(out, &DorisVectorEmbedding{
|
||||
ID: id,
|
||||
Content: content,
|
||||
SourceID: sourceID,
|
||||
SourceType: sourceType,
|
||||
ChunkID: chunkID,
|
||||
KnowledgeID: knowledgeID,
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
TagID: tagID,
|
||||
IsEnabled: isEnabled,
|
||||
Embedding: vec,
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildRetrieveResult 把 IndexWithScore 列表包装成 RetrieveResult。
|
||||
func buildRetrieveResult(results []*types.IndexWithScore, retrieverType types.RetrieverType) []*types.RetrieveResult {
|
||||
return []*types.RetrieveResult{{
|
||||
Results: results,
|
||||
RetrieverEngineType: types.DorisRetrieverEngineType,
|
||||
RetrieverType: retrieverType,
|
||||
Error: nil,
|
||||
}}
|
||||
}
|
||||
|
||||
// calculateStorageSize 估算单行的存储成本。
|
||||
//
|
||||
// 与 Qdrant 一致:payload 字符串字节 + 向量 (dim*4) + HNSW M*2*8 + 元数据 24。
|
||||
func calculateStorageSize(emb *DorisVectorEmbedding) int64 {
|
||||
var payload int64
|
||||
payload += int64(len(emb.Content))
|
||||
payload += int64(len(emb.SourceID))
|
||||
payload += int64(len(emb.ChunkID))
|
||||
payload += int64(len(emb.KnowledgeID))
|
||||
payload += int64(len(emb.KnowledgeBaseID))
|
||||
payload += int64(len(emb.TagID))
|
||||
payload += 8 // source_type int
|
||||
|
||||
var vec int64
|
||||
var hnsw int64
|
||||
if len(emb.Embedding) > 0 {
|
||||
vec = int64(len(emb.Embedding)) * 4
|
||||
const hnswM = 32
|
||||
hnsw = hnswM * 2 * 8
|
||||
}
|
||||
const metaBytes int64 = 24
|
||||
return payload + vec + hnsw + metaBytes
|
||||
}
|
||||
@@ -0,0 +1,510 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newTestRepo 构造一个共享 sqlmock 的 dorisRepository,
|
||||
// 默认绕过 ensureTable(initializedTables 已置位),便于专注测试 SQL 形态。
|
||||
//
|
||||
// 返回的 cleanup 用 defer 调用即可。
|
||||
func newTestRepo(t *testing.T) (*dorisRepository, sqlmock.Sqlmock, *httptest.Server, func()) {
|
||||
t.Helper()
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// 默认行为:成功,回 1 行。
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
_ = body
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Status": "Success",
|
||||
"NumberTotalRows": 1,
|
||||
"NumberLoadedRows": 1,
|
||||
"Label": "test",
|
||||
})
|
||||
}))
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
httpClient: srv.Client(),
|
||||
feHTTPBase: srv.URL,
|
||||
username: "u",
|
||||
password: "p",
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
_ = db.Close()
|
||||
srv.Close()
|
||||
}
|
||||
return repo, mock, srv, cleanup
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// query.go:whereBuilder / embeddingLiteral / parseEmbeddingLiteral
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestEmbeddingLiteralRoundTrip(t *testing.T) {
|
||||
t.Run("empty vector returns []", func(t *testing.T) {
|
||||
assert.Equal(t, "[]", embeddingLiteral(nil))
|
||||
assert.Equal(t, "[]", embeddingLiteral([]float32{}))
|
||||
})
|
||||
|
||||
t.Run("does not contain locale-sensitive separators", func(t *testing.T) {
|
||||
s := embeddingLiteral([]float32{1.5, -2.25, 0.001})
|
||||
// 必须只包含数字 / 点 / 负号 / e / 逗号 / 方括号
|
||||
assert.Regexp(t, regexp.MustCompile(`^\[[\-+0-9eE.,]+\]$`), s)
|
||||
})
|
||||
|
||||
t.Run("round trip", func(t *testing.T) {
|
||||
orig := []float32{1.5, -2.25, 0.0625}
|
||||
s := embeddingLiteral(orig)
|
||||
got, err := parseEmbeddingLiteral([]byte(s))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, orig, got)
|
||||
})
|
||||
|
||||
t.Run("parse handles whitespace and missing brackets", func(t *testing.T) {
|
||||
v, err := parseEmbeddingLiteral([]byte(" 1.0 , 2.0 ,3.0 "))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []float32{1.0, 2.0, 3.0}, v)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWhereBuilder(t *testing.T) {
|
||||
t.Run("empty builder returns 1 = 1", func(t *testing.T) {
|
||||
w := &whereBuilder{}
|
||||
clause, args := w.build()
|
||||
assert.Equal(t, "1 = 1", clause)
|
||||
assert.Nil(t, args)
|
||||
})
|
||||
|
||||
t.Run("equal + IN + NOT IN", func(t *testing.T) {
|
||||
w := &whereBuilder{}
|
||||
w.addEqual("is_enabled", true)
|
||||
w.addIn("knowledge_base_id", []string{"kb1", "kb2"})
|
||||
w.addNotIn("chunk_id", []string{"x"})
|
||||
clause, args := w.build()
|
||||
assert.Contains(t, clause, "is_enabled = ?")
|
||||
assert.Contains(t, clause, "knowledge_base_id IN (?, ?)")
|
||||
assert.Contains(t, clause, "chunk_id NOT IN (?)")
|
||||
assert.Equal(t, []any{true, "kb1", "kb2", "x"}, args)
|
||||
})
|
||||
|
||||
t.Run("buildBaseFilter applies all params", func(t *testing.T) {
|
||||
w := buildBaseFilter(types.RetrieveParams{
|
||||
KnowledgeBaseIDs: []string{"kb1"},
|
||||
KnowledgeIDs: []string{"k1", "k2"},
|
||||
TagIDs: []string{"t1"},
|
||||
ExcludeKnowledgeIDs: []string{"k9"},
|
||||
ExcludeChunkIDs: []string{"c9"},
|
||||
})
|
||||
clause, _ := w.build()
|
||||
assert.Contains(t, clause, "is_enabled = ?")
|
||||
assert.Contains(t, clause, "knowledge_base_id IN (?)")
|
||||
assert.Contains(t, clause, "knowledge_id IN (?, ?)")
|
||||
assert.Contains(t, clause, "tag_id IN (?)")
|
||||
assert.Contains(t, clause, "knowledge_id NOT IN (?)")
|
||||
assert.Contains(t, clause, "chunk_id NOT IN (?)")
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// streamload.go:chunkRows / partialUpdateRows
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestChunkRows(t *testing.T) {
|
||||
rows := []map[string]any{
|
||||
{"id": "a", "is_enabled": true},
|
||||
{"id": "b", "is_enabled": false},
|
||||
{"id": "c", "is_enabled": true},
|
||||
}
|
||||
|
||||
t.Run("single batch when fits", func(t *testing.T) {
|
||||
batches := chunkRows(rows, 4096)
|
||||
require.Len(t, batches, 1)
|
||||
assert.Len(t, batches[0], 3)
|
||||
})
|
||||
|
||||
t.Run("splits when exceeding maxBytes", func(t *testing.T) {
|
||||
// 给一个非常小的上限,强制每行单独成段
|
||||
batches := chunkRows(rows, 16)
|
||||
assert.GreaterOrEqual(t, len(batches), 2)
|
||||
|
||||
var total int
|
||||
for _, b := range batches {
|
||||
total += len(b)
|
||||
}
|
||||
assert.Equal(t, len(rows), total)
|
||||
})
|
||||
}
|
||||
|
||||
func TestPartialUpdateRows_HappyPath(t *testing.T) {
|
||||
repo, _, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
// 自定义 server 验证请求形态。
|
||||
var captured *http.Request
|
||||
var capturedBody []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
captured = r
|
||||
capturedBody, _ = io.ReadAll(r.Body)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Status": "Success",
|
||||
"NumberTotalRows": 2,
|
||||
"NumberLoadedRows": 2,
|
||||
"Label": "ok",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
repo.feHTTPBase = srv.URL
|
||||
|
||||
rows := []map[string]any{
|
||||
{"id": "id1", "is_enabled": true},
|
||||
{"id": "id2", "is_enabled": false},
|
||||
}
|
||||
require.NoError(t, repo.partialUpdateRows(context.Background(),
|
||||
"weknora_embeddings_768", []string{"id", "is_enabled"}, rows))
|
||||
|
||||
require.NotNil(t, captured)
|
||||
assert.Equal(t, http.MethodPut, captured.Method)
|
||||
assert.Equal(t, "/api/weknora/weknora_embeddings_768/_stream_load", captured.URL.Path)
|
||||
|
||||
assert.Equal(t, "true", captured.Header.Get("partial_columns"))
|
||||
assert.Equal(t, "true", captured.Header.Get("strip_outer_array"))
|
||||
assert.Equal(t, "json", captured.Header.Get("format"))
|
||||
assert.Equal(t, "id,is_enabled", captured.Header.Get("columns"))
|
||||
assert.Equal(t, "APPEND", captured.Header.Get("merge_type"))
|
||||
assert.True(t, strings.HasPrefix(captured.Header.Get("Authorization"), "Basic "))
|
||||
|
||||
var got []map[string]any
|
||||
require.NoError(t, json.Unmarshal(capturedBody, &got))
|
||||
assert.Len(t, got, 2)
|
||||
assert.Equal(t, "id1", got[0]["id"])
|
||||
}
|
||||
|
||||
func TestPartialUpdateRows_FailureSurfaced(t *testing.T) {
|
||||
repo, _, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Status": "Fail",
|
||||
"Message": "label exists",
|
||||
"ErrorURL": "http://...",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
repo.feHTTPBase = srv.URL
|
||||
|
||||
err := repo.partialUpdateRows(context.Background(),
|
||||
"t", []string{"id", "is_enabled"},
|
||||
[]map[string]any{{"id": "x", "is_enabled": true}},
|
||||
)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "stream load failed")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// repository.go:SQL 形态
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestDeleteByChunkIDList_SQLShape(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
mock.ExpectExec(`DELETE FROM .*weknora_embeddings_768.* WHERE chunk_id IN \(\?, \?\)`).
|
||||
WithArgs("c1", "c2").
|
||||
WillReturnResult(sqlmock.NewResult(0, 2))
|
||||
|
||||
err := repo.DeleteByChunkIDList(context.Background(), []string{"c1", "c2"}, 768, "")
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestDeleteByKnowledgeIDList_NoOpOnEmpty(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
require.NoError(t, repo.DeleteByKnowledgeIDList(context.Background(), nil, 768, ""))
|
||||
// 空列表不应触发任何 query
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestVectorRetrieve_SQLShape(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings_3").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1))
|
||||
|
||||
mock.ExpectQuery(`SELECT id, content, .*cosine_distance_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`).
|
||||
WithArgs(true, 0.5, 5).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{
|
||||
"id", "content", "source_id", "source_type",
|
||||
"chunk_id", "knowledge_id", "knowledge_base_id", "tag_id",
|
||||
"is_enabled", "score",
|
||||
}).AddRow("id1", "hello", "src", 0, "c1", "k1", "kb1", "t1", true, 0.95),
|
||||
)
|
||||
|
||||
results, err := repo.VectorRetrieve(context.Background(), types.RetrieveParams{
|
||||
Embedding: []float32{1, 2, 3},
|
||||
TopK: 5,
|
||||
Threshold: 0.5,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
require.Len(t, results[0].Results, 1)
|
||||
assert.Equal(t, "id1", results[0].Results[0].ID)
|
||||
assert.InDelta(t, 0.95, results[0].Results[0].Score, 1e-9)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestKeywordsRetrieve_SQLShape(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).
|
||||
AddRow("weknora_embeddings_768"))
|
||||
|
||||
mock.ExpectQuery(`MATCH_ANY \?`).
|
||||
WithArgs(true, "你好", 3).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{
|
||||
"id", "content", "source_id", "source_type",
|
||||
"chunk_id", "knowledge_id", "knowledge_base_id", "tag_id",
|
||||
"is_enabled",
|
||||
}).AddRow("id1", "你好世界", "src", 0, "c1", "k1", "kb1", "", true),
|
||||
)
|
||||
|
||||
results, err := repo.KeywordsRetrieve(context.Background(), types.RetrieveParams{
|
||||
Query: "你好",
|
||||
TopK: 3,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1)
|
||||
require.Len(t, results[0].Results, 1)
|
||||
assert.Equal(t, "id1", results[0].Results[0].ID)
|
||||
assert.Equal(t, 1.0, results[0].Results[0].Score)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBatchUpdateChunkEnabledStatus_StreamLoad(t *testing.T) {
|
||||
// 这条路径覆盖了 lookupChunkRowKeys + partialUpdateRows 的连贯流。
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
var streamLoadCalls int
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
streamLoadCalls++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"Status": "Success",
|
||||
"NumberLoadedRows": 1,
|
||||
"Label": "label",
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
httpClient: srv.Client(),
|
||||
feHTTPBase: srv.URL,
|
||||
username: "u",
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
|
||||
mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings\\_%").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).AddRow("weknora_embeddings_768"))
|
||||
mock.ExpectQuery(`SELECT id, chunk_id FROM .*weknora_embeddings_768.* WHERE chunk_id IN`).
|
||||
WithArgs("c1").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"id", "chunk_id"}).AddRow("row-1", "c1"))
|
||||
|
||||
require.NoError(t, repo.BatchUpdateChunkEnabledStatus(
|
||||
context.Background(), map[string]bool{"c1": false}))
|
||||
assert.Equal(t, 1, streamLoadCalls, "stream load should be invoked once")
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestEnsureTable_DDLShape(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
bucketsNum: 5,
|
||||
replicationNum: 2,
|
||||
}
|
||||
|
||||
// 表不存在
|
||||
mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`).
|
||||
WithArgs("weknora", "weknora_embeddings_768").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0))
|
||||
// CREATE TABLE 应包含关键属性
|
||||
mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*UNIQUE KEY\(id\).*BUCKETS 5.*replication_num.*=.*2.*enable_unique_key_merge_on_write.*=.*true`).
|
||||
WillReturnResult(sqlmock.NewResult(0, 0))
|
||||
// SHOW INDEX 一次即返回 ANN 已 FINISHED
|
||||
mock.ExpectQuery(`SHOW INDEX FROM .*weknora_embeddings_768.*`).
|
||||
WillReturnRows(
|
||||
sqlmock.NewRows([]string{"Table", "Key_name", "State"}).
|
||||
AddRow("weknora_embeddings_768", "idx_emb", "FINISHED"),
|
||||
)
|
||||
|
||||
require.NoError(t, repo.ensureTable(context.Background(), 768))
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
func TestBatchSave_SQLShape(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
database: "weknora",
|
||||
tableBaseName: "weknora_embeddings",
|
||||
}
|
||||
repo.initializedTables.Store(3, true) // 跳过 ensureTable
|
||||
|
||||
mock.ExpectExec(`INSERT INTO .*weknora_embeddings_3.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`).
|
||||
WithArgs(
|
||||
"src1", "hello", "src1", 0,
|
||||
"c1", "k1", "kb1", "",
|
||||
true,
|
||||
).
|
||||
WillReturnResult(sqlmock.NewResult(0, 1))
|
||||
|
||||
err = repo.BatchSave(context.Background(),
|
||||
[]*types.IndexInfo{{
|
||||
Content: "hello",
|
||||
SourceID: "src1",
|
||||
ChunkID: "c1",
|
||||
KnowledgeID: "k1",
|
||||
KnowledgeBaseID: "kb1",
|
||||
IsEnabled: true,
|
||||
}},
|
||||
map[string]any{
|
||||
fieldEmbedding: map[string][]float32{
|
||||
"src1": {1, 2, 3},
|
||||
},
|
||||
},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Engine wiring
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestEngineTypeAndSupport(t *testing.T) {
|
||||
repo, _, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
assert.Equal(t, types.DorisRetrieverEngineType, repo.EngineType())
|
||||
supports := repo.Support()
|
||||
assert.Contains(t, supports, types.KeywordsRetrieverType)
|
||||
assert.Contains(t, supports, types.VectorRetrieverType)
|
||||
}
|
||||
|
||||
func TestRetrieve_DispatchesByType(t *testing.T) {
|
||||
repo, mock, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
// invalid retriever type -> error,不会触发任何 SQL
|
||||
_, err := repo.Retrieve(context.Background(), types.RetrieveParams{
|
||||
RetrieverType: "unknown",
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// regression / utility
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestTranslateSourceID(t *testing.T) {
|
||||
t.Run("plain chunk uses target chunk", func(t *testing.T) {
|
||||
got := translateSourceID("c1", "c1", "tc1")
|
||||
assert.Equal(t, "tc1", got)
|
||||
})
|
||||
t.Run("generated question preserves question id", func(t *testing.T) {
|
||||
got := translateSourceID("c1-q9", "c1", "tc1")
|
||||
assert.Equal(t, "tc1-q9", got)
|
||||
})
|
||||
t.Run("unrecognized source falls back to fresh uuid", func(t *testing.T) {
|
||||
got := translateSourceID("totally-other", "c1", "tc1")
|
||||
assert.NotEqual(t, "totally-other", got)
|
||||
assert.Len(t, got, 36) // UUID 长度
|
||||
})
|
||||
}
|
||||
|
||||
func TestEstimateStorageSize(t *testing.T) {
|
||||
repo, _, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
out := repo.EstimateStorageSize(context.Background(),
|
||||
[]*types.IndexInfo{{Content: "hello", ChunkID: "c1", KnowledgeID: "k1", KnowledgeBaseID: "kb1"}},
|
||||
map[string]any{
|
||||
fieldEmbedding: map[string][]float32{
|
||||
"": {1, 2, 3},
|
||||
},
|
||||
},
|
||||
)
|
||||
assert.Greater(t, out, int64(0))
|
||||
}
|
||||
|
||||
// 保证 *sql.Rows 错误不会被吞掉。
|
||||
func TestScanRetrieveRows_Error(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
mock.ExpectQuery("SELECT 1").
|
||||
WillReturnError(errors.New("boom"))
|
||||
|
||||
rows, err := db.Query("SELECT 1")
|
||||
if err != nil {
|
||||
// query 直接失败也算预期
|
||||
assert.Equal(t, "boom", err.Error())
|
||||
return
|
||||
}
|
||||
_, scanErr := scanRetrieveRows(rows, types.MatchTypeEmbedding)
|
||||
if scanErr != nil {
|
||||
assert.Error(t, scanErr)
|
||||
}
|
||||
}
|
||||
|
||||
// silence unused import warning when sql isn't directly used
|
||||
var _ = sql.ErrNoRows
|
||||
@@ -0,0 +1,280 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// 默认的桶数 / 副本数。Doris 在 PROPERTIES 不指定时会用集群默认值,
|
||||
// 这里给一个对单机/小集群更友好的保守值。
|
||||
const (
|
||||
defaultBucketsNum = 10
|
||||
defaultReplicationNum = 1
|
||||
|
||||
// ANN 索引就绪轮询的最大等待时间。索引未就绪不阻塞写入路径,
|
||||
// 只阻塞 ensureTable 自身(首次建表场景),所以 30s 是可接受的。
|
||||
annReadyTimeout = 30 * time.Second
|
||||
annReadyPoll = 1 * time.Second
|
||||
)
|
||||
|
||||
// getTableName 返回某个维度对应的物理表名:<base>_<dim>。
|
||||
//
|
||||
// 与 Qdrant/Milvus/Weaviate 的 collection 命名约定一致,
|
||||
// 这样不同 embedding 模型(不同维度)的数据互不冲突。
|
||||
func (r *dorisRepository) getTableName(dimension int) string {
|
||||
return fmt.Sprintf("%s_%d", r.tableBaseName, dimension)
|
||||
}
|
||||
|
||||
// ensureTable 保证目标维度对应的表已经存在;
|
||||
// 不存在则用 CREATE TABLE IF NOT EXISTS 创建,并在创建后轮询 ANN 索引就绪。
|
||||
//
|
||||
// 该方法在每次 Save / BatchSave 之前调用,结果缓存在 initializedTables 中,
|
||||
// 同一进程内同一 dimension 只会真正打一次 SHOW TABLES + DDL。
|
||||
func (r *dorisRepository) ensureTable(ctx context.Context, dimension int) error {
|
||||
if _, ok := r.initializedTables.Load(dimension); ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
log := logger.GetLogger(ctx)
|
||||
tableName := r.getTableName(dimension)
|
||||
|
||||
exists, err := r.tableExists(ctx, tableName)
|
||||
if err != nil {
|
||||
log.Errorf("[Doris] Failed to check table existence: %v", err)
|
||||
return fmt.Errorf("check table existence: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
log.Infof("[Doris] Creating table %s with dimension %d", tableName, dimension)
|
||||
if err := r.createTable(ctx, tableName, dimension); err != nil {
|
||||
log.Errorf("[Doris] Failed to create table: %v", err)
|
||||
return fmt.Errorf("create table: %w", err)
|
||||
}
|
||||
|
||||
// 创建表后等一会让 ANN 索引就绪,超时不致命。
|
||||
if err := r.waitANNReady(ctx, tableName); err != nil {
|
||||
log.Warnf("[Doris] ANN index for %s not ready within %s: %v "+
|
||||
"(writes will still proceed; queries may fall back to brute force temporarily)",
|
||||
tableName, annReadyTimeout, err)
|
||||
}
|
||||
}
|
||||
|
||||
r.initializedTables.Store(dimension, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// tableExists 通过 information_schema 判断表是否存在。
|
||||
//
|
||||
// 不直接用 SHOW TABLES 是因为 Doris 4.1 对 SHOW TABLES LIKE 大小写敏感,
|
||||
// 而 information_schema 与 MySQL 兼容性更好。
|
||||
func (r *dorisRepository) tableExists(ctx context.Context, tableName string) (bool, error) {
|
||||
const q = `SELECT COUNT(1) FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?`
|
||||
var n int
|
||||
if err := r.db.QueryRowContext(ctx, q, r.database, tableName).Scan(&n); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// createTable 发出 CREATE TABLE DDL。Doris DDL 是同步的(除 ANN 索引构建外),
|
||||
// 返回成功即代表表已可写。
|
||||
func (r *dorisRepository) createTable(ctx context.Context, tableName string, dimension int) error {
|
||||
buckets := r.bucketsNum
|
||||
if buckets <= 0 {
|
||||
buckets = defaultBucketsNum
|
||||
}
|
||||
replication := r.replicationNum
|
||||
if replication <= 0 {
|
||||
replication = defaultReplicationNum
|
||||
}
|
||||
|
||||
ddl := buildCreateTableDDL(tableName, dimension, buckets, replication)
|
||||
_, err := r.db.ExecContext(ctx, ddl)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildCreateTableDDL 根据维度生成 CREATE TABLE DDL。
|
||||
//
|
||||
// 关键点:
|
||||
// - UNIQUE KEY(id) + enable_unique_key_merge_on_write=true:
|
||||
// 这是 partial update(Stream Load) 起作用的前提。
|
||||
// - INVERTED 索引覆盖所有过滤字段 + 中文分词的 content 全文索引。
|
||||
// - ANN 索引使用 HNSW + cosine_distance,与 Qdrant/Weaviate 的 cosine 相似度方向一致。
|
||||
//
|
||||
// 注意:DDL 中 dimension / buckets / replication 三个数值字段是 Go 端格式化拼接的,
|
||||
// 不存在 SQL 注入风险(来源都是受控的 IndexConfig int)。
|
||||
func buildCreateTableDDL(tableName string, dimension, buckets, replication int) string {
|
||||
const tpl = `CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` (
|
||||
id VARCHAR(64) NOT NULL,
|
||||
chunk_id VARCHAR(64),
|
||||
knowledge_id VARCHAR(64),
|
||||
knowledge_base_id VARCHAR(64),
|
||||
source_id VARCHAR(255),
|
||||
source_type INT,
|
||||
tag_id VARCHAR(64),
|
||||
is_enabled BOOLEAN,
|
||||
content TEXT,
|
||||
embedding ARRAY<FLOAT> NOT NULL,
|
||||
INDEX idx_chunk (chunk_id) USING INVERTED,
|
||||
INDEX idx_kb (knowledge_base_id) USING INVERTED,
|
||||
INDEX idx_kid (knowledge_id) USING INVERTED,
|
||||
INDEX idx_src (source_id) USING INVERTED,
|
||||
INDEX idx_tag (tag_id) USING INVERTED,
|
||||
INDEX idx_enabled (is_enabled) USING INVERTED,
|
||||
INDEX idx_content (content) USING INVERTED PROPERTIES("parser"="chinese","support_phrase"="true"),
|
||||
INDEX idx_emb (embedding) USING ANN PROPERTIES(
|
||||
"index_type"="hnsw",
|
||||
"metric_type"="cosine_distance",
|
||||
"dim"="%d",
|
||||
"max_degree"="32",
|
||||
"ef_construction"="200"
|
||||
)
|
||||
) ENGINE=OLAP
|
||||
UNIQUE KEY(id)
|
||||
DISTRIBUTED BY HASH(id) BUCKETS %d
|
||||
PROPERTIES(
|
||||
"replication_num"="%d",
|
||||
"enable_unique_key_merge_on_write"="true"
|
||||
);`
|
||||
return fmt.Sprintf(tpl, tableName, dimension, buckets, replication)
|
||||
}
|
||||
|
||||
// waitANNReady 轮询 SHOW INDEX,等待 ANN 索引进入 FINISHED 状态。
|
||||
//
|
||||
// Doris 的 ANN 索引在建表后会异步构建,期间查询会退化为 brute-force(结果对,速度慢)。
|
||||
// 此处仅做"尽力而为"的等待:到点未就绪只记 warning,不阻塞写入。
|
||||
func (r *dorisRepository) waitANNReady(ctx context.Context, tableName string) error {
|
||||
deadline := time.Now().Add(annReadyTimeout)
|
||||
for {
|
||||
ready, err := r.annIndexReady(ctx, tableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ready {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("ann index not ready within %s", annReadyTimeout)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(annReadyPoll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// annIndexReady 检查 ANN 索引的 State 是否为 FINISHED。
|
||||
//
|
||||
// SHOW INDEX FROM <table> 在 Doris 上返回多列;不同小版本列序略有差异,
|
||||
// 这里以列名匹配(来自 information_schema.statistics + 自定义 view 不可行,
|
||||
// 直接用 SHOW INDEX 然后扫描即可)。
|
||||
//
|
||||
// 兼容策略:如果 SHOW INDEX 返回中找不到 idx_emb 行(极旧版本),视为已就绪,
|
||||
// 避免因为不同 Doris 版本的输出差异把启动卡死。
|
||||
func (r *dorisRepository) annIndexReady(ctx context.Context, tableName string) (bool, error) {
|
||||
rows, err := r.db.QueryContext(ctx,
|
||||
fmt.Sprintf("SHOW INDEX FROM `%s`", tableName))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
keyNameIdx, stateIdx := -1, -1
|
||||
for i, c := range cols {
|
||||
switch strings.ToLower(c) {
|
||||
case "key_name":
|
||||
keyNameIdx = i
|
||||
case "state", "index_state":
|
||||
stateIdx = i
|
||||
}
|
||||
}
|
||||
|
||||
foundANN := false
|
||||
for rows.Next() {
|
||||
// 使用 sql.RawBytes 接收以兼容不同列类型。
|
||||
raw := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range raw {
|
||||
ptrs[i] = &raw[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
var keyName, state string
|
||||
if keyNameIdx >= 0 {
|
||||
keyName = bytesToString(raw[keyNameIdx])
|
||||
}
|
||||
if stateIdx >= 0 {
|
||||
state = bytesToString(raw[stateIdx])
|
||||
}
|
||||
|
||||
if keyName != "idx_emb" {
|
||||
continue
|
||||
}
|
||||
foundANN = true
|
||||
if stateIdx < 0 {
|
||||
// 旧版本不暴露 state 列,乐观认为已就绪。
|
||||
return true, nil
|
||||
}
|
||||
if !strings.EqualFold(state, "FINISHED") &&
|
||||
!strings.EqualFold(state, "NORMAL") {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if !foundANN {
|
||||
// 没找到 ANN 行(可能是 Doris 版本返回字段不同),不阻塞。
|
||||
return true, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// listEmbeddingTables 返回当前 database 下所有 <base>_% 命名的表,
|
||||
// 用于关键词检索 / 跨维度 BatchUpdate。
|
||||
func (r *dorisRepository) listEmbeddingTables(ctx context.Context) ([]string, error) {
|
||||
const q = `SELECT TABLE_NAME FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = ? AND TABLE_NAME LIKE ?`
|
||||
rows, err := r.db.QueryContext(ctx, q, r.database, r.tableBaseName+"\\_%")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var names []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
return names, rows.Err()
|
||||
}
|
||||
|
||||
// bytesToString 把 SHOW INDEX 返回的 raw any(通常是 []byte 或 string)
|
||||
// 安全转成字符串。
|
||||
func bytesToString(v any) string {
|
||||
switch s := v.(type) {
|
||||
case []byte:
|
||||
return string(s)
|
||||
case string:
|
||||
return s
|
||||
case nil:
|
||||
return ""
|
||||
default:
|
||||
return fmt.Sprintf("%v", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,345 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// Stream Load 相关常量。
|
||||
const (
|
||||
// 单批 Stream Load 的 JSON body 上限(保守值,远小于 Doris 默认 streaming_load_max_mb=10240)。
|
||||
// 主要目的是控制单次 HTTP 请求的尾延迟,超过则自动拆批。
|
||||
streamLoadMaxBatchBytes = 1 << 20 // 1 MiB
|
||||
|
||||
// HTTP 头 Authorization 用 Basic auth;Stream Load 也支持 token,
|
||||
// 这里走最常用的用户名/密码方案与 MySQL 协议保持一致。
|
||||
headerAuthorization = "Authorization"
|
||||
headerExpect = "Expect"
|
||||
headerContentType = "Content-Type"
|
||||
)
|
||||
|
||||
// streamLoadResponse 是 Doris FE/BE 返回的 Stream Load 结果体。
|
||||
//
|
||||
// 关键字段:Status 应为 "Success" 或 "Publish Timeout"(后者表示数据已写入但发布事务超时,
|
||||
// 仍视为成功)。其它状态都视为失败。
|
||||
type streamLoadResponse struct {
|
||||
TxnId int64 `json:"TxnId"`
|
||||
Label string `json:"Label"`
|
||||
Status string `json:"Status"`
|
||||
Message string `json:"Message"`
|
||||
NumberTotalRows int64 `json:"NumberTotalRows"`
|
||||
NumberLoadedRows int64 `json:"NumberLoadedRows"`
|
||||
NumberFilteredRows int64 `json:"NumberFilteredRows"`
|
||||
NumberUnselectedRows int64 `json:"NumberUnselectedRows"`
|
||||
LoadBytes int64 `json:"LoadBytes"`
|
||||
LoadTimeMs int64 `json:"LoadTimeMs"`
|
||||
BeginTxnTimeMs int64 `json:"BeginTxnTimeMs"`
|
||||
StreamLoadPutTimeMs int64 `json:"StreamLoadPutTimeMs"`
|
||||
ReadDataTimeMs int64 `json:"ReadDataTimeMs"`
|
||||
WriteDataTimeMs int64 `json:"WriteDataTimeMs"`
|
||||
CommitAndPublishTimeMs int64 `json:"CommitAndPublishTimeMs"`
|
||||
ErrorURL string `json:"ErrorURL"`
|
||||
}
|
||||
|
||||
// streamLoadURL 拼装某张表的 Stream Load HTTP 端点。
|
||||
func (r *dorisRepository) streamLoadURL(table string) string {
|
||||
return fmt.Sprintf("%s/api/%s/%s/_stream_load",
|
||||
r.feHTTPBase, r.database, table)
|
||||
}
|
||||
|
||||
// partialUpdateRows 把若干行通过 Stream Load 的 partial update 模式写回目标表。
|
||||
//
|
||||
// columns 是参与本次 partial update 的列(必须包含 UNIQUE KEY 列,即 "id")。
|
||||
// rows 中每一项是一个与 columns 等长的字段值数组。
|
||||
//
|
||||
// 实现要点:
|
||||
// 1. 用 JSON 数组的 body 形式,header 加 strip_outer_array=true。
|
||||
// 2. 设置 partial_columns=true、merge_type=APPEND,触发 Doris 的 partial update 模式
|
||||
// (Doris 4.1 + UNIQUE KEY MoW 表的标准玩法)。
|
||||
// 3. 按 streamLoadMaxBatchBytes 自动拆批,避免单次过大。
|
||||
// 4. 处理 307:Doris 的 FE 会 redirect 到 BE,net/http 默认会跟随;
|
||||
// 这里需要保证 GetBody 可重发(已通过 bytes.NewReader 构造 Body 满足)。
|
||||
func (r *dorisRepository) partialUpdateRows(ctx context.Context,
|
||||
table string, columns []string, rows []map[string]any,
|
||||
) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, batch := range chunkRows(rows, streamLoadMaxBatchBytes) {
|
||||
if err := r.streamLoadOnce(ctx, table, columns, batch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// streamLoadOnce 发出一次 Stream Load HTTP 请求。
|
||||
func (r *dorisRepository) streamLoadOnce(ctx context.Context,
|
||||
table string, columns []string, rows []map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
body, err := json.Marshal(rows)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal stream load body: %w", err)
|
||||
}
|
||||
|
||||
url := r.streamLoadURL(table)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("build stream load request: %w", err)
|
||||
}
|
||||
|
||||
// GetBody 让 redirect 时可以重新读 body(FE -> BE 的 307 需要重发 PUT body)。
|
||||
bodyCopy := append([]byte(nil), body...)
|
||||
req.GetBody = func() (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(bodyCopy)), nil
|
||||
}
|
||||
req.ContentLength = int64(len(body))
|
||||
|
||||
auth := base64.StdEncoding.EncodeToString([]byte(r.username + ":" + r.password))
|
||||
req.Header.Set(headerAuthorization, "Basic "+auth)
|
||||
req.Header.Set(headerExpect, "100-continue")
|
||||
req.Header.Set(headerContentType, "application/json")
|
||||
req.Header.Set("format", "json")
|
||||
req.Header.Set("strip_outer_array", "true")
|
||||
req.Header.Set("partial_columns", "true")
|
||||
req.Header.Set("columns", strings.Join(columns, ","))
|
||||
req.Header.Set("merge_type", "APPEND")
|
||||
|
||||
resp, err := r.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stream load HTTP: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read stream load response: %w", err)
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("stream load HTTP %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var result streamLoadResponse
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return fmt.Errorf("decode stream load response: %w (raw=%s)", err, string(respBody))
|
||||
}
|
||||
|
||||
switch result.Status {
|
||||
case "Success", "Publish Timeout":
|
||||
log.Infof("[Doris] Stream load %s OK: rows=%d, loaded=%d, label=%s",
|
||||
table, result.NumberTotalRows, result.NumberLoadedRows, result.Label)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("stream load failed: status=%s msg=%s err_url=%s",
|
||||
result.Status, result.Message, result.ErrorURL)
|
||||
}
|
||||
}
|
||||
|
||||
// chunkRows 把行按累积 JSON 体大小切分,每段不超过 maxBytes。
|
||||
//
|
||||
// 注意:JSON 序列化的实际开销大约是 marshal 后的字节数,而单行 marshal
|
||||
// 加上逗号 + 数组括号约等于本估算。这里用粗略估计避免每段都 marshal。
|
||||
func chunkRows(rows []map[string]any, maxBytes int) [][]map[string]any {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
out [][]map[string]any
|
||||
curr []map[string]any
|
||||
size int
|
||||
header = 2 // "[" + "]"
|
||||
)
|
||||
|
||||
for _, row := range rows {
|
||||
raw, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
// marshal 失败时把这一行单独成段,由上层 streamLoadOnce 再次 marshal 报错。
|
||||
if len(curr) > 0 {
|
||||
out = append(out, curr)
|
||||
}
|
||||
out = append(out, []map[string]any{row})
|
||||
curr = nil
|
||||
size = 0
|
||||
continue
|
||||
}
|
||||
// 加上逗号位(除第一行之外)。
|
||||
need := len(raw)
|
||||
if len(curr) > 0 {
|
||||
need++
|
||||
}
|
||||
if size+need+header > maxBytes && len(curr) > 0 {
|
||||
out = append(out, curr)
|
||||
curr = nil
|
||||
size = 0
|
||||
}
|
||||
curr = append(curr, row)
|
||||
size += need
|
||||
}
|
||||
if len(curr) > 0 {
|
||||
out = append(out, curr)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 业务面方法:BatchUpdateChunkEnabledStatus / BatchUpdateChunkTagID
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// BatchUpdateChunkEnabledStatus 批量更新 chunk 的 is_enabled 字段。
|
||||
//
|
||||
// 由于 Stream Load partial update 需要主键列 id,但 chunk_id 不一定等于 id,
|
||||
// 这里做一个查表步骤:通过 chunk_id IN (...) 查出对应的 (id, table) 列表,
|
||||
// 再分表用 partial update 写回。
|
||||
//
|
||||
// 时间复杂度上是 O(N):扫描一遍 chunk_id IN (...) 在 INVERTED 索引下走点查。
|
||||
func (r *dorisRepository) BatchUpdateChunkEnabledStatus(ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
if len(chunkStatusMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, 0, len(chunkStatusMap))
|
||||
for id := range chunkStatusMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
|
||||
mapping, err := r.lookupChunkRowKeys(ctx, chunkIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 按表分组 -> 每行 (id, is_enabled)。
|
||||
byTable := make(map[string][]map[string]any)
|
||||
for chunkID, locations := range mapping {
|
||||
enabled, ok := chunkStatusMap[chunkID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, loc := range locations {
|
||||
byTable[loc.table] = append(byTable[loc.table], map[string]any{
|
||||
fieldID: loc.id,
|
||||
fieldIsEnabled: enabled,
|
||||
})
|
||||
}
|
||||
}
|
||||
for table, rows := range byTable {
|
||||
if err := r.partialUpdateRows(ctx, table, []string{fieldID, fieldIsEnabled}, rows); err != nil {
|
||||
return fmt.Errorf("partial update is_enabled in %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchUpdateChunkTagID 批量更新 chunk 的 tag_id 字段。逻辑与 EnabledStatus 一致。
|
||||
func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context,
|
||||
chunkTagMap map[string]string,
|
||||
) error {
|
||||
if len(chunkTagMap) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
chunkIDs := make([]string, 0, len(chunkTagMap))
|
||||
for id := range chunkTagMap {
|
||||
chunkIDs = append(chunkIDs, id)
|
||||
}
|
||||
|
||||
mapping, err := r.lookupChunkRowKeys(ctx, chunkIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
byTable := make(map[string][]map[string]any)
|
||||
for chunkID, locations := range mapping {
|
||||
tagID, ok := chunkTagMap[chunkID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, loc := range locations {
|
||||
byTable[loc.table] = append(byTable[loc.table], map[string]any{
|
||||
fieldID: loc.id,
|
||||
fieldTagID: tagID,
|
||||
})
|
||||
}
|
||||
}
|
||||
for table, rows := range byTable {
|
||||
if err := r.partialUpdateRows(ctx, table, []string{fieldID, fieldTagID}, rows); err != nil {
|
||||
return fmt.Errorf("partial update tag_id in %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rowLocation 表示某行在哪个表里、主键 id 是什么。
|
||||
type rowLocation struct {
|
||||
table string
|
||||
id string
|
||||
}
|
||||
|
||||
// lookupChunkRowKeys 查询给定的 chunkIDs 在所有 <base>_<dim> 表中的物理位置:
|
||||
// - key:chunk_id
|
||||
// - value:[(table, id), ...],因为同一 chunk 可能在多个维度的表里都有副本。
|
||||
//
|
||||
// 跨表查询使用 listEmbeddingTables 列出的所有匹配表;每张表执行一次
|
||||
// SELECT id, chunk_id FROM <table> WHERE chunk_id IN (?, ?, ...)。
|
||||
func (r *dorisRepository) lookupChunkRowKeys(ctx context.Context,
|
||||
chunkIDs []string,
|
||||
) (map[string][]rowLocation, error) {
|
||||
if len(chunkIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tables, err := r.listEmbeddingTables(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tables: %w", err)
|
||||
}
|
||||
if len(tables) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
placeholders := make([]string, len(chunkIDs))
|
||||
args := make([]any, len(chunkIDs))
|
||||
for i, v := range chunkIDs {
|
||||
placeholders[i] = "?"
|
||||
args[i] = v
|
||||
}
|
||||
|
||||
out := make(map[string][]rowLocation)
|
||||
for _, table := range tables {
|
||||
stmt := fmt.Sprintf(
|
||||
"SELECT %s, %s FROM `%s` WHERE %s IN (%s)",
|
||||
fieldID, fieldChunkID, table, fieldChunkID, strings.Join(placeholders, ", "),
|
||||
)
|
||||
rows, err := r.db.QueryContext(ctx, stmt, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("lookup chunk row keys in %s: %w", table, err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var id, chunkID string
|
||||
if err := rows.Scan(&id, &chunkID); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, fmt.Errorf("scan row keys: %w", err)
|
||||
}
|
||||
out[chunkID] = append(out[chunkID], rowLocation{table: table, id: id})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
_ = rows.Close()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package doris
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// dorisRepository 是 Apache Doris 4.1 的检索引擎仓储实现。
|
||||
//
|
||||
// 通信通道:
|
||||
// - 读写主链路:MySQL 协议(database/sql + go-sql-driver/mysql),FE 默认 9030 端口。
|
||||
// - Stream Load:HTTP(FE 默认 8030 端口),用于 BatchUpdate* 的 partial update。
|
||||
//
|
||||
// 表结构按维度分表:<tableBaseName>_<dim>,UNIQUE KEY(id),
|
||||
// 同时建立倒排索引(filter / 全文)+ ANN(HNSW) 索引。
|
||||
//
|
||||
// 与 Qdrant/Milvus/Weaviate 一样,initializedTables 缓存"已确保存在"的维度,
|
||||
// 避免每次写入都打 SHOW TABLES。
|
||||
type dorisRepository struct {
|
||||
db *sql.DB
|
||||
|
||||
httpClient *http.Client
|
||||
// fe HTTP base,例如 "http://doris-fe:8030"。Stream Load 路径
|
||||
// 由 streamLoadURL(table) 拼接:<feHTTPBase>/api/<database>/<table>/_stream_load。
|
||||
feHTTPBase string
|
||||
|
||||
username string
|
||||
password string
|
||||
database string
|
||||
|
||||
tableBaseName string
|
||||
bucketsNum int // 0 -> default 10
|
||||
replicationNum int // 0 -> default 1
|
||||
|
||||
// 已经确保过 ensureTable 的维度集合:dim -> true。
|
||||
initializedTables sync.Map
|
||||
}
|
||||
|
||||
// DorisVectorEmbedding 是落到 Doris 表里的一行的领域模型。
|
||||
//
|
||||
// 字段顺序与 schema.go 中的 INSERT 列序保持一致,
|
||||
// 调整时需要同时更新 createInsert 与 columns。
|
||||
type DorisVectorEmbedding struct {
|
||||
ID string
|
||||
Content string
|
||||
SourceID string
|
||||
SourceType int
|
||||
ChunkID string
|
||||
KnowledgeID string
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
IsEnabled bool
|
||||
Embedding []float32
|
||||
}
|
||||
|
||||
// DorisVectorEmbeddingWithScore 是检索结果的领域模型,
|
||||
// Score 在向量检索时存储 (1 - cosine_distance_approximate),
|
||||
// 在关键词检索时统一赋 1.0(与 Qdrant 行为一致)。
|
||||
type DorisVectorEmbeddingWithScore struct {
|
||||
DorisVectorEmbedding
|
||||
Score float64
|
||||
}
|
||||
@@ -182,6 +182,13 @@ func validateConnectionConfig(engineType types.RetrieverEngineType, config types
|
||||
if config.Host == "" {
|
||||
return errors.NewValidationError("host is required for weaviate")
|
||||
}
|
||||
case types.DorisRetrieverEngineType:
|
||||
if config.Addr == "" {
|
||||
return errors.NewValidationError("addr is required for doris (FE MySQL host:port)")
|
||||
}
|
||||
if config.Database == "" {
|
||||
return errors.NewValidationError("database is required for doris")
|
||||
}
|
||||
case types.SQLiteRetrieverEngineType:
|
||||
// No connection config needed for SQLite
|
||||
}
|
||||
|
||||
@@ -13,7 +13,8 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql
|
||||
_ "github.com/go-sql-driver/mysql" // MySQL driver for database/sql, used by Doris connection test
|
||||
_ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate/auth"
|
||||
@@ -40,6 +41,8 @@ func (s *vectorStoreService) TestConnection(
|
||||
return testMilvusConnection(ctx, config)
|
||||
case types.WeaviateRetrieverEngineType:
|
||||
return testWeaviateConnection(ctx, config)
|
||||
case types.DorisRetrieverEngineType:
|
||||
return testDorisConnection(ctx, config)
|
||||
case types.SQLiteRetrieverEngineType:
|
||||
// SQLite is file-based, no remote connection to test
|
||||
return "", nil
|
||||
@@ -226,3 +229,43 @@ func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig)
|
||||
|
||||
return meta.Version, nil
|
||||
}
|
||||
|
||||
// testDorisConnection 通过 MySQL 协议(database/sql + go-sql-driver)
|
||||
// Ping Doris FE 并查询 @@version。
|
||||
//
|
||||
// Doris 4.1 的 @@version 形如 "5.7.99 Doris-4.1.0"——前半段是兼容性表达式,
|
||||
// 后半段才是真正的 Doris 版本号。这里直接返回原样字符串,由调用方按需展示。
|
||||
func testDorisConnection(ctx context.Context, config types.ConnectionConfig) (string, error) {
|
||||
testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
if config.Addr == "" {
|
||||
return "", errors.NewBadRequestError("failed to create doris connection: addr is required")
|
||||
}
|
||||
|
||||
// Database 不强制要求;Ping 时无明确库则用 information_schema(任何 MySQL 兼容服务都有)。
|
||||
database := config.Database
|
||||
if database == "" {
|
||||
database = "information_schema"
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?timeout=5s",
|
||||
config.Username, config.Password, config.Addr, database)
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
return "", errors.NewBadRequestError("failed to create doris connection: invalid configuration")
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.PingContext(testCtx); err != nil {
|
||||
logger.Warnf(ctx, "Doris connection test failed: %v", err)
|
||||
return "", errors.NewBadRequestError("failed to connect to doris: connection refused or authentication failed")
|
||||
}
|
||||
|
||||
var version string
|
||||
if err := db.QueryRowContext(testCtx, "SELECT @@version").Scan(&version); err != nil {
|
||||
logger.Warnf(ctx, "Doris version detection failed: %v", err)
|
||||
return "", nil
|
||||
}
|
||||
return version, nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
@@ -659,6 +660,31 @@ func TestTestConnection_PostgresDefaultConnection(t *testing.T) {
|
||||
assert.Empty(t, version) // default connection cannot detect version without DB handle
|
||||
}
|
||||
|
||||
func TestTestConnection_DorisInvalidAddr(t *testing.T) {
|
||||
// 给一个不可达的地址 + 5s timeout,期望返回 BadRequestError 而非 panic。
|
||||
repo := &mockVectorStoreRepo{}
|
||||
svc := NewVectorStoreService(repo, nil, nil)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := svc.TestConnection(ctx, types.DorisRetrieverEngineType, types.ConnectionConfig{
|
||||
Addr: "127.0.0.1:1", // 一定不可连通
|
||||
Database: "weknora",
|
||||
Username: "root",
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestTestConnection_DorisMissingAddr(t *testing.T) {
|
||||
repo := &mockVectorStoreRepo{}
|
||||
svc := NewVectorStoreService(repo, nil, nil)
|
||||
|
||||
_, err := svc.TestConnection(context.Background(), types.DorisRetrieverEngineType,
|
||||
types.ConnectionConfig{})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// validateConnectionConfig tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -742,6 +768,24 @@ func TestValidateConnectionConfig(t *testing.T) {
|
||||
config: types.ConnectionConfig{},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "doris valid",
|
||||
engineType: types.DorisRetrieverEngineType,
|
||||
config: types.ConnectionConfig{Addr: "doris-fe:9030", Database: "weknora"},
|
||||
wantError: false,
|
||||
},
|
||||
{
|
||||
name: "doris missing addr",
|
||||
engineType: types.DorisRetrieverEngineType,
|
||||
config: types.ConnectionConfig{Database: "weknora"},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "doris missing database",
|
||||
engineType: types.DorisRetrieverEngineType,
|
||||
config: types.ConnectionConfig{Addr: "doris-fe:9030"},
|
||||
wantError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
Reference in New Issue
Block a user