diff --git a/.env.example b/.env.example index 42ef3c20..5627fc6b 100644 --- a/.env.example +++ b/.env.example @@ -545,6 +545,9 @@ DOCREADER_TRANSPORT=grpc # Doris 表前缀,对应每维度的物理表 _ # DORIS_TABLE_PREFIX=weknora_embeddings +# Doris compatibility mode: auto (probe server), legacy (cosine_distance), or inner_product_duplicate (inner_product); NOT interchangeable after tables are created. +# DORIS_COMPAT_MODE= + # Tavily Search API Key(可选,启用 Tavily 网页搜索提供者) # TAVILY_API_KEY=tvly-your_tavily_api_key diff --git a/docker-compose.yml b/docker-compose.yml index 2049b148..d06ab8b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -120,6 +120,7 @@ services: - DORIS_USERNAME=${DORIS_USERNAME:-root} - DORIS_PASSWORD=${DORIS_PASSWORD:-} - DORIS_TABLE_PREFIX=${DORIS_TABLE_PREFIX:-weknora_embeddings} + - DORIS_COMPAT_MODE=${DORIS_COMPAT_MODE:-auto} - STORAGE_TYPE=${STORAGE_TYPE:-} - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-} - AUTO_RECOVER_DIRTY=${AUTO_RECOVER_DIRTY:-true} diff --git a/internal/application/repository/retriever/doris/compat.go b/internal/application/repository/retriever/doris/compat.go new file mode 100644 index 00000000..3877befd --- /dev/null +++ b/internal/application/repository/retriever/doris/compat.go @@ -0,0 +1,228 @@ +package doris + +import ( + "context" + "database/sql" + "fmt" + "os" + "sort" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" +) + +const envDorisCompatMode = "DORIS_COMPAT_MODE" + +type dorisCompatMode string + +const ( + dorisCompatModeAuto dorisCompatMode = "auto" + dorisCompatModeLegacy dorisCompatMode = "legacy" + dorisCompatModeInnerProductDuplicate dorisCompatMode = "inner_product_duplicate" +) + +type dorisCompatProbe struct { + innerProductApproximate bool + cosineDistanceApproximate bool +} + +func resolveConfiguredDorisCompatMode() (mode dorisCompatMode, invalid string) { + raw := strings.TrimSpace(os.Getenv(envDorisCompatMode)) + if raw == "" { + return dorisCompatModeAuto, "" + } + + switch strings.ToLower(raw) { + case string(dorisCompatModeAuto): + return dorisCompatModeAuto, "" + case string(dorisCompatModeLegacy): + return dorisCompatModeLegacy, "" + case string(dorisCompatModeInnerProductDuplicate), "inner-product-duplicate", "inner_product", "inner-product": + return dorisCompatModeInnerProductDuplicate, "" + default: + return dorisCompatModeAuto, raw + } +} + +func (r *dorisRepository) resolveCompatMode(ctx context.Context) (dorisCompatMode, error) { + r.compatResolveOnce.Do(func() { + log := logger.GetLogger(ctx) + requested := r.compatModeRequested + if requested == "" { + r.compatModeResolved = dorisCompatModeInnerProductDuplicate + return + } + + existingMode, exampleTable, foundExisting, err := r.detectExistingCompatMode(ctx) + if err != nil { + r.compatResolveErr = err + return + } + + if foundExisting { + if requested != dorisCompatModeAuto && requested != existingMode { + r.compatResolveErr = fmt.Errorf( + "Doris compat mode %q does not match existing embedding tables (detected %q from %s). %s is not interchangeable after %s_* tables are created. Recreate the existing %s_* tables before switching modes, or set %s=%s", + requested, + existingMode, + exampleTable, + envDorisCompatMode, + r.tableBaseName, + r.tableBaseName, + envDorisCompatMode, + existingMode, + ) + log.Errorf("[Doris] %v", r.compatResolveErr) + return + } + + r.compatModeResolved = existingMode + log.Warnf( + "[Doris] Using compat mode %s from existing embedding tables (%s). %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.", + existingMode, + exampleTable, + envDorisCompatMode, + r.tableBaseName, + ) + return + } + + if requested == dorisCompatModeAuto { + probe, probeErr := r.probeCompatMode(ctx) + if probeErr != nil { + r.compatResolveErr = probeErr + log.Errorf("[Doris] %v", probeErr) + return + } + log.Infof( + "[Doris] Compat probe result: inner_product_approximate=%t, cosine_distance_approximate=%t", + probe.innerProductApproximate, + probe.cosineDistanceApproximate, + ) + + switch { + case probe.innerProductApproximate: + r.compatModeResolved = dorisCompatModeInnerProductDuplicate + case probe.cosineDistanceApproximate: + r.compatModeResolved = dorisCompatModeLegacy + default: + r.compatResolveErr = fmt.Errorf( + "Doris compatibility auto-detection could not find a supported vector function. Set %s=%s or %s=%s explicitly after verifying your Doris build. %s is not interchangeable after %s_* tables are created", + envDorisCompatMode, + dorisCompatModeInnerProductDuplicate, + envDorisCompatMode, + dorisCompatModeLegacy, + envDorisCompatMode, + r.tableBaseName, + ) + log.Errorf("[Doris] %v", r.compatResolveErr) + return + } + + log.Warnf( + "[Doris] Auto-selected compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.", + r.compatModeResolved, + envDorisCompatMode, + r.tableBaseName, + ) + return + } + + r.compatModeResolved = requested + log.Warnf( + "[Doris] Using configured compat mode %s for new embedding tables. %s is not interchangeable after %s_* tables are created; recreate those tables before switching modes.", + r.compatModeResolved, + envDorisCompatMode, + r.tableBaseName, + ) + }) + + return r.compatModeResolved, r.compatResolveErr +} + +func (r *dorisRepository) detectExistingCompatMode(ctx context.Context) (dorisCompatMode, string, bool, error) { + tables, err := r.listEmbeddingTables(ctx) + if err != nil { + return "", "", false, fmt.Errorf("list Doris embedding tables: %w", err) + } + if len(tables) == 0 { + return "", "", false, nil + } + sort.Strings(tables) + + var detected dorisCompatMode + for _, table := range tables { + ddl, err := r.showCreateTable(ctx, table) + if err != nil { + return "", "", false, fmt.Errorf("show create table %s: %w", table, err) + } + mode, err := compatModeFromDDL(ddl) + if err != nil { + return "", "", false, fmt.Errorf("detect compat mode from %s: %w", table, err) + } + if detected == "" { + detected = mode + continue + } + if detected != mode { + return "", "", false, fmt.Errorf( + "existing Doris embedding tables use mixed compat modes (%s and %s). %s is not interchangeable after %s_* tables are created; recreate the existing %s_* tables with a single mode", + detected, + mode, + envDorisCompatMode, + r.tableBaseName, + r.tableBaseName, + ) + } + } + + return detected, tables[0], true, nil +} + +func (r *dorisRepository) showCreateTable(ctx context.Context, table string) (string, error) { + row := r.db.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE `%s`", table)) + var tableName, ddl string + if err := row.Scan(&tableName, &ddl); err != nil { + return "", err + } + return ddl, nil +} + +func compatModeFromDDL(ddl string) (dorisCompatMode, error) { + normalized := strings.ToLower(ddl) + switch { + case strings.Contains(normalized, "duplicate key("): + return dorisCompatModeInnerProductDuplicate, nil + case strings.Contains(normalized, "unique key("): + return dorisCompatModeLegacy, nil + default: + return "", fmt.Errorf("unsupported table definition; expected UNIQUE KEY or DUPLICATE KEY") + } +} + +func (r *dorisRepository) probeCompatMode(ctx context.Context) (dorisCompatProbe, error) { + probe := dorisCompatProbe{} + probe.innerProductApproximate = r.vectorFunctionSupported(ctx, "inner_product_approximate([1.0],[1.0])") + probe.cosineDistanceApproximate = r.vectorFunctionSupported(ctx, "cosine_distance_approximate([1.0],[1.0])") + return probe, nil +} + +func (r *dorisRepository) vectorFunctionSupported(ctx context.Context, expr string) bool { + var value sql.NullFloat64 + if err := r.db.QueryRowContext(ctx, "SELECT "+expr).Scan(&value); err != nil { + return false + } + return true +} + +func (m dorisCompatMode) normalizeEmbeddings() bool { + return m != dorisCompatModeLegacy +} + +func (m dorisCompatMode) usesReplaceWrite() bool { + return m != dorisCompatModeLegacy +} + +func (m dorisCompatMode) usesRewriteChunkUpdates() bool { + return m != dorisCompatModeLegacy +} diff --git a/internal/application/repository/retriever/doris/query.go b/internal/application/repository/retriever/doris/query.go index 3c159a72..766d3160 100644 --- a/internal/application/repository/retriever/doris/query.go +++ b/internal/application/repository/retriever/doris/query.go @@ -189,6 +189,28 @@ func validateEmbedding(vec []float32) error { return nil } +// normalizeEmbedding returns a unit-length copy of vec so Doris inner-product +// ANN search can preserve cosine-style similarity semantics. +func normalizeEmbedding(vec []float32) []float32 { + if len(vec) == 0 { + return nil + } + var sumSquares float64 + for _, value := range vec { + f := float64(value) + sumSquares += f * f + } + if sumSquares == 0 { + return append([]float32(nil), vec...) + } + norm := float32(math.Sqrt(sumSquares)) + normalized := make([]float32, len(vec)) + for i, value := range vec { + normalized[i] = value / norm + } + return normalized +} + // errInvalidEmbedding 描述哪个下标含非有限值;用结构体而非 fmt.Errorf // 是为了让上层在日志里能拿到下标做问题定位。 type errInvalidEmbedding struct { diff --git a/internal/application/repository/retriever/doris/repository.go b/internal/application/repository/retriever/doris/repository.go index e755682e..121b8b8b 100644 --- a/internal/application/repository/retriever/doris/repository.go +++ b/internal/application/repository/retriever/doris/repository.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "net/http" + "os" "strings" "github.com/Tencent/WeKnora/internal/logger" @@ -35,6 +36,10 @@ func NewDorisRetrieveEngineRepository( log.Info("[Doris] Initializing Doris retriever engine repository") tableBaseName := types.ResolveCollectionName(indexCfg, envDorisTablePrefix, defaultTableBaseName) + compatMode, invalidCompatMode := resolveConfiguredDorisCompatMode() + if invalidCompatMode != "" { + log.Warnf("[Doris] Invalid %s=%q, defaulting to %s", envDorisCompatMode, invalidCompatMode, dorisCompatModeAuto) + } repo := &dorisRepository{ db: db, @@ -46,9 +51,13 @@ func NewDorisRetrieveEngineRepository( tableBaseName: tableBaseName, bucketsNum: indexCfg.GetBucketsNum(0), replicationNum: indexCfg.GetReplicationNum(0), + compatModeRequested: compatMode, + } + log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s, compat_mode=%s", + database, tableBaseName, repo.feHTTPBase, repo.compatModeRequested) + if os.Getenv(envDorisCompatMode) == "" { + log.Infof("[Doris] %s not set, defaulting to %s and probing on first use", envDorisCompatMode, dorisCompatModeAuto) } - log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s", - database, tableBaseName, repo.feHTTPBase) return repo } @@ -68,7 +77,7 @@ func (r *dorisRepository) EstimateStorageSize(_ context.Context, ) int64 { var total int64 for _, info := range indexInfoList { - emb := toDorisVectorEmbedding(info, params) + emb := toDorisVectorEmbedding(info, params, dorisCompatModeInnerProductDuplicate) total += calculateStorageSize(emb) } return total @@ -81,8 +90,8 @@ func (r *dorisRepository) Save(ctx context.Context, return r.BatchSave(ctx, []*types.IndexInfo{info}, additionalParams) } -// BatchSave 把同一批 IndexInfo 按维度分组,对每个维度构造一条 -// INSERT INTO ... VALUES (...), (...) 语句。UNIQUE KEY 表会自动按 id upsert。 +// BatchSave 把同一批 IndexInfo 按维度分组;Doris ANN 兼容表使用 +// DUPLICATE KEY,因此这里显式按 id 做 delete + insert,保持原先的替换语义。 func (r *dorisRepository) BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any, ) error { @@ -90,10 +99,14 @@ func (r *dorisRepository) BatchSave(ctx context.Context, if len(indexInfoList) == 0 { return nil } + compatMode, err := r.resolveCompatMode(ctx) + if err != nil { + return err + } groups := make(map[int][]*DorisVectorEmbedding) for _, info := range indexInfoList { - emb := toDorisVectorEmbedding(info, additionalParams) + emb := toDorisVectorEmbedding(info, additionalParams, compatMode) if len(emb.Embedding) == 0 { log.Warnf("[Doris] Skipping empty embedding for chunk %s", info.ChunkID) continue @@ -117,7 +130,12 @@ func (r *dorisRepository) BatchSave(ctx context.Context, if err := r.ensureTable(ctx, dim); err != nil { return err } - if err := r.insertRows(ctx, r.getTableName(dim), rows); err != nil { + if compatMode.usesReplaceWrite() { + err = r.replaceRows(ctx, r.getTableName(dim), rows) + } else { + err = r.insertRows(ctx, r.getTableName(dim), rows) + } + if err != nil { return fmt.Errorf("batch save dim=%d: %w", dim, err) } log.Infof("[Doris] Saved %d rows to %s", len(rows), r.getTableName(dim)) @@ -157,6 +175,63 @@ func (r *dorisRepository) insertRows(ctx context.Context, return err } +// replaceRows 在 DUPLICATE KEY 表上显式模拟“按 id 覆盖”的语义。 +func (r *dorisRepository) replaceRows(ctx context.Context, + table string, rows []*DorisVectorEmbedding, +) error { + rows = dedupeRowsByID(rows) + if len(rows) == 0 { + return nil + } + + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + if err := r.deleteRowsByID(ctx, table, ids); err != nil { + return err + } + return r.insertRows(ctx, table, rows) +} + +func (r *dorisRepository) deleteRowsByID(ctx context.Context, + table string, ids []string, +) error { + if len(ids) == 0 { + return nil + } + + placeholders := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + + stmt := fmt.Sprintf("DELETE FROM `%s` WHERE %s IN (%s)", + table, fieldID, strings.Join(placeholders, ", ")) + _, err := r.db.ExecContext(ctx, stmt, args...) + return err +} + +func dedupeRowsByID(rows []*DorisVectorEmbedding) []*DorisVectorEmbedding { + if len(rows) < 2 { + return rows + } + + out := make([]*DorisVectorEmbedding, 0, len(rows)) + positions := make(map[string]int, len(rows)) + for _, row := range rows { + if idx, ok := positions[row.ID]; ok { + out[idx] = row + continue + } + positions[row.ID] = len(out) + out = append(out, row) + } + return out +} + // DeleteByChunkIDList 用 chunk_id 列删除。dimension 用于定位具体表。 func (r *dorisRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, _ string, @@ -219,8 +294,9 @@ func (r *dorisRepository) Retrieve(ctx context.Context, return nil, fmt.Errorf("invalid retriever type: %v", params.RetrieverType) } -// VectorRetrieve 调用 cosine_distance_approximate 做 ANN 搜索, -// score = 1 - distance 与 Qdrant cosine 相似度方向一致:值越大越相似。 +// VectorRetrieve 对查询向量先做单位化,再调用 inner_product_approximate 做 +// ANN 搜索;对单位向量而言,inner product 与 cosine similarity 等价, +// 因此 score 仍然保持“越大越相似”的语义。 func (r *dorisRepository) VectorRetrieve(ctx context.Context, params types.RetrieveParams, ) ([]*types.RetrieveResult, error) { @@ -228,6 +304,14 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context, if err := validateEmbedding(params.Embedding); err != nil { return nil, fmt.Errorf("invalid query embedding: %w", err) } + compatMode, err := r.resolveCompatMode(ctx) + if err != nil { + return nil, err + } + queryEmbedding := append([]float32(nil), params.Embedding...) + if compatMode.normalizeEmbeddings() { + queryEmbedding = normalizeEmbedding(queryEmbedding) + } dim := len(params.Embedding) table := r.getTableName(dim) @@ -245,14 +329,17 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context, // embedding 必须用字面量,threshold/topK 用占位符。 // 使用 HAVING 是因为 score 是 SELECT 列别名,WHERE 阶段还看不到。 + scoreExpr := fmt.Sprintf("inner_product_approximate(`%s`, %s)", fieldEmbedding, embeddingLiteral(queryEmbedding)) + if compatMode == dorisCompatModeLegacy { + scoreExpr = fmt.Sprintf("(1 - cosine_distance_approximate(`%s`, %s))", fieldEmbedding, embeddingLiteral(queryEmbedding)) + } stmt := fmt.Sprintf( - "SELECT %s, (1 - cosine_distance_approximate(`%s`, %s)) AS score "+ + "SELECT %s, %s AS score "+ "FROM `%s` WHERE %s "+ "HAVING score >= ? "+ "ORDER BY score DESC LIMIT ?", strings.Join(columnsForRetrieve, ", "), - fieldEmbedding, - embeddingLiteral(params.Embedding), + scoreExpr, table, whereClause, ) @@ -260,7 +347,7 @@ func (r *dorisRepository) VectorRetrieve(ctx context.Context, rows, err := r.db.QueryContext(ctx, stmt, args...) if err != nil { - return nil, fmt.Errorf("vector retrieve %s: %w", table, err) + return nil, r.wrapVectorRetrieveError(table, compatMode, err) } defer rows.Close() @@ -417,7 +504,7 @@ func (r *dorisRepository) CopyIndices(ctx context.Context, } // BatchUpdateChunkEnabledStatus / BatchUpdateChunkTagID 实际实现位于 streamload.go, -// 通过 Stream Load partial update 协议执行高性能字段级更新。 +// 会按 compat mode 选择 partial update 或 rewrite rows。 // --------------------------------------------------------------------------- // 私有辅助 @@ -426,7 +513,12 @@ func (r *dorisRepository) CopyIndices(ctx context.Context, // toDorisVectorEmbedding 把 IndexInfo + 上层传入的 embedding 映射 转换为 // Doris 行模型。Embedding 通过 additionalParams[fieldEmbedding] 中的 // map[string][]float32 按 SourceID 取出,与 Qdrant/Milvus 完全一致。 -func toDorisVectorEmbedding(info *types.IndexInfo, additionalParams map[string]any) *DorisVectorEmbedding { +// inner_product_duplicate 模式会先单位化,legacy 模式保留原始向量。 +func toDorisVectorEmbedding( + info *types.IndexInfo, + additionalParams map[string]any, + compatMode dorisCompatMode, +) *DorisVectorEmbedding { emb := &DorisVectorEmbedding{ ID: info.ID, Content: info.Content, @@ -441,13 +533,32 @@ func toDorisVectorEmbedding(info *types.IndexInfo, additionalParams map[string]a if additionalParams != nil { if v, ok := additionalParams[fieldEmbedding]; ok { if m, ok := v.(map[string][]float32); ok { - emb.Embedding = m[info.SourceID] + emb.Embedding = append([]float32(nil), m[info.SourceID]...) + if compatMode.normalizeEmbeddings() { + emb.Embedding = normalizeEmbedding(emb.Embedding) + } } } } return emb } +func (r *dorisRepository) wrapVectorRetrieveError(table string, compatMode dorisCompatMode, err error) error { + if compatMode == dorisCompatModeLegacy { + return fmt.Errorf( + "vector retrieve %s in Doris compat mode %s: %w. If your Doris build does not support cosine_distance_approximate or ANN on UNIQUE KEY tables, set %s=%s before creating embedding tables. %s is not interchangeable after %s_* tables are created", + table, + compatMode, + err, + envDorisCompatMode, + dorisCompatModeInnerProductDuplicate, + envDorisCompatMode, + r.tableBaseName, + ) + } + return fmt.Errorf("vector retrieve %s in Doris compat mode %s: %w", table, compatMode, err) +} + // translateSourceID 把源 SourceID 翻译到目标 SourceID,与 Qdrant 实现完全镜像: // - 普通 chunk:SourceID == ChunkID -> 使用 targetChunkID // - 生成型问题:SourceID == "-" -> "-" diff --git a/internal/application/repository/retriever/doris/repository_test.go b/internal/application/repository/retriever/doris/repository_test.go index ec86e9bb..34f1b1c6 100644 --- a/internal/application/repository/retriever/doris/repository_test.go +++ b/internal/application/repository/retriever/doris/repository_test.go @@ -58,6 +58,12 @@ func newTestRepo(t *testing.T) (*dorisRepository, sqlmock.Sqlmock, *httptest.Ser return repo, mock, srv, cleanup } +func primeCompatMode(repo *dorisRepository, mode dorisCompatMode, err error) { + repo.compatModeResolved = mode + repo.compatResolveErr = err + repo.compatResolveOnce.Do(func() {}) +} + // --------------------------------------------------------------------------- // query.go:whereBuilder / embeddingLiteral / parseEmbeddingLiteral // --------------------------------------------------------------------------- @@ -259,7 +265,7 @@ func TestVectorRetrieve_SQLShape(t *testing.T) { WithArgs("weknora", "weknora_embeddings_3"). WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) - mock.ExpectQuery(`SELECT id, content, .*cosine_distance_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`). + mock.ExpectQuery(`SELECT id, content, .*inner_product_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`). WithArgs(true, 0.5, 5). WillReturnRows( sqlmock.NewRows([]string{ @@ -283,6 +289,53 @@ func TestVectorRetrieve_SQLShape(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestVectorRetrieve_SQLShape_LegacyMode(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + primeCompatMode(repo, dorisCompatModeLegacy, nil) + + mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`). + WithArgs("weknora", "weknora_embeddings_3"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(1)) + + mock.ExpectQuery(`SELECT id, content, .*cosine_distance_approximate.*HAVING score >= \? ORDER BY score DESC LIMIT \?`). + WithArgs(true, 0.5, 5). + WillReturnRows( + sqlmock.NewRows([]string{ + "id", "content", "source_id", "source_type", + "chunk_id", "knowledge_id", "knowledge_base_id", "tag_id", + "is_enabled", "score", + }).AddRow("id1", "hello", "src", 0, "c1", "k1", "kb1", "t1", true, 0.8), + ) + + results, err := repo.VectorRetrieve(context.Background(), types.RetrieveParams{ + Embedding: []float32{1, 2, 3}, + TopK: 5, + Threshold: 0.5, + RetrieverType: types.VectorRetrieverType, + }) + require.NoError(t, err) + require.Len(t, results, 1) + require.Len(t, results[0].Results, 1) + assert.Equal(t, "id1", results[0].Results[0].ID) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestNormalizeEmbedding(t *testing.T) { + t.Run("returns unit vector copy", func(t *testing.T) { + got := normalizeEmbedding([]float32{3, 4}) + require.Len(t, got, 2) + assert.InDelta(t, 0.6, got[0], 1e-6) + assert.InDelta(t, 0.8, got[1], 1e-6) + }) + + t.Run("keeps zero vector finite", func(t *testing.T) { + got := normalizeEmbedding([]float32{0, 0}) + require.Len(t, got, 2) + assert.Equal(t, []float32{0, 0}, got) + }) +} + func TestKeywordsRetrieve_SQLShape(t *testing.T) { repo, mock, _, cleanup := newTestRepo(t) defer cleanup() @@ -315,32 +368,44 @@ func TestKeywordsRetrieve_SQLShape(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } -func TestBatchUpdateChunkEnabledStatus_StreamLoad(t *testing.T) { - // 这条路径覆盖了 lookupChunkRowKeys + partialUpdateRows 的连贯流。 +func TestBatchUpdateChunkEnabledStatus_RewritesRows(t *testing.T) { db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) require.NoError(t, err) defer db.Close() - var streamLoadCalls int - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - streamLoadCalls++ - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]any{ - "Status": "Success", - "NumberLoadedRows": 1, - "Label": "label", - }) - })) - defer srv.Close() - repo := &dorisRepository{ db: db, - httpClient: srv.Client(), - feHTTPBase: srv.URL, - username: "u", database: "weknora", tableBaseName: "weknora_embeddings", } + primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil) + + mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`). + WithArgs("weknora", "weknora_embeddings\\_%"). + WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).AddRow("weknora_embeddings_768")) + mock.ExpectQuery(`SELECT id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id, is_enabled, embedding FROM .*weknora_embeddings_768.* WHERE chunk_id IN`). + WithArgs("c1"). + WillReturnRows(sqlmock.NewRows([]string{ + "id", "content", "source_id", "source_type", + "chunk_id", "knowledge_id", "knowledge_base_id", "tag_id", + "is_enabled", "embedding", + }).AddRow("row-1", "hello", "src1", 0, "c1", "k1", "kb1", "t1", true, "[1,2,3]")) + mock.ExpectExec(`DELETE FROM .*weknora_embeddings_768.* WHERE id IN \(\?\)`). + WithArgs("row-1"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO .*weknora_embeddings_768.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`). + WithArgs("row-1", "hello", "src1", 0, "c1", "k1", "kb1", "t1", false). + WillReturnResult(sqlmock.NewResult(0, 1)) + + require.NoError(t, repo.BatchUpdateChunkEnabledStatus( + context.Background(), map[string]bool{"c1": false})) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestBatchUpdateChunkEnabledStatus_LegacyModeUsesPartialUpdate(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + primeCompatMode(repo, dorisCompatModeLegacy, nil) mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`). WithArgs("weknora", "weknora_embeddings\\_%"). @@ -351,7 +416,6 @@ func TestBatchUpdateChunkEnabledStatus_StreamLoad(t *testing.T) { require.NoError(t, repo.BatchUpdateChunkEnabledStatus( context.Background(), map[string]bool{"c1": false})) - assert.Equal(t, 1, streamLoadCalls, "stream load should be invoked once") assert.NoError(t, mock.ExpectationsWereMet()) } @@ -367,13 +431,14 @@ func TestEnsureTable_DDLShape(t *testing.T) { bucketsNum: 5, replicationNum: 2, } + primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil) // 表不存在 mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`). WithArgs("weknora", "weknora_embeddings_768"). WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) - // CREATE TABLE 应包含关键属性 - mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*UNIQUE KEY\(id\).*BUCKETS 5.*replication_num.*=.*2.*enable_unique_key_merge_on_write.*=.*true`). + // CREATE TABLE 应包含关键属性和 Doris 支持的 inner_product ANN metric + mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*metric_type"="inner_product".*DUPLICATE KEY\(id\).*BUCKETS 5.*replication_num.*=.*2`). WillReturnResult(sqlmock.NewResult(0, 0)) // SHOW INDEX 一次即返回 ANN 已 FINISHED mock.ExpectQuery(`SHOW INDEX FROM .*weknora_embeddings_768.*`). @@ -389,6 +454,37 @@ func TestEnsureTable_DDLShape(t *testing.T) { }, 2*time.Second, 10*time.Millisecond, "expectations should be met after async ANN poll") } +func TestEnsureTable_DDLShape_LegacyMode(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + require.NoError(t, err) + defer db.Close() + + repo := &dorisRepository{ + db: db, + database: "weknora", + tableBaseName: "weknora_embeddings", + bucketsNum: 5, + replicationNum: 2, + } + primeCompatMode(repo, dorisCompatModeLegacy, nil) + + mock.ExpectQuery(`SELECT COUNT\(1\) FROM information_schema.tables`). + WithArgs("weknora", "weknora_embeddings_768"). + WillReturnRows(sqlmock.NewRows([]string{"c"}).AddRow(0)) + mock.ExpectExec(`CREATE TABLE IF NOT EXISTS .*weknora_embeddings_768.*metric_type"="cosine_distance".*UNIQUE KEY\(id\).*enable_unique_key_merge_on_write.*true`). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery(`SHOW INDEX FROM .*weknora_embeddings_768.*`). + WillReturnRows( + sqlmock.NewRows([]string{"Table", "Key_name", "State"}). + AddRow("weknora_embeddings_768", "idx_emb", "FINISHED"), + ) + + require.NoError(t, repo.ensureTable(context.Background(), 768)) + require.Eventually(t, func() bool { + return mock.ExpectationsWereMet() == nil + }, 2*time.Second, 10*time.Millisecond, "expectations should be met after async ANN poll") +} + func TestBatchSave_SQLShape(t *testing.T) { db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) require.NoError(t, err) @@ -399,8 +495,52 @@ func TestBatchSave_SQLShape(t *testing.T) { database: "weknora", tableBaseName: "weknora_embeddings", } + primeCompatMode(repo, dorisCompatModeInnerProductDuplicate, nil) repo.initializedTables.Store(3, true) // 跳过 ensureTable + mock.ExpectExec(`DELETE FROM .*weknora_embeddings_3.* WHERE id IN \(\?\)`). + WithArgs("src1"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(`INSERT INTO .*weknora_embeddings_3.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`). + WithArgs( + "src1", "hello", "src1", 0, + "c1", "k1", "kb1", "", + true, + ). + WillReturnResult(sqlmock.NewResult(0, 1)) + + err = repo.BatchSave(context.Background(), + []*types.IndexInfo{{ + Content: "hello", + SourceID: "src1", + ChunkID: "c1", + KnowledgeID: "k1", + KnowledgeBaseID: "kb1", + IsEnabled: true, + }}, + map[string]any{ + fieldEmbedding: map[string][]float32{ + "src1": {1, 2, 3}, + }, + }, + ) + require.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestBatchSave_SQLShape_LegacyMode(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + require.NoError(t, err) + defer db.Close() + + repo := &dorisRepository{ + db: db, + database: "weknora", + tableBaseName: "weknora_embeddings", + } + primeCompatMode(repo, dorisCompatModeLegacy, nil) + repo.initializedTables.Store(3, true) + mock.ExpectExec(`INSERT INTO .*weknora_embeddings_3.*VALUES \(\?, \?, \?, \?, \?, \?, \?, \?, \?, \[`). WithArgs( "src1", "hello", "src1", 0, @@ -428,6 +568,46 @@ func TestBatchSave_SQLShape(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestResolveCompatMode_RejectsExistingModeSwitch(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + repo.compatModeRequested = dorisCompatModeLegacy + + mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`). + WithArgs("weknora", "weknora_embeddings\\_%"). + WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"}).AddRow("weknora_embeddings_768")) + mock.ExpectQuery(`SHOW CREATE TABLE .*weknora_embeddings_768.*`). + WillReturnRows(sqlmock.NewRows([]string{"Table", "Create Table"}).AddRow( + "weknora_embeddings_768", + "CREATE TABLE `weknora_embeddings_768` (id VARCHAR(64)) ENGINE=OLAP DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 10 PROPERTIES(\"replication_num\"=\"1\")", + )) + + _, err := repo.resolveCompatMode(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "not interchangeable after weknora_embeddings_*") + assert.Contains(t, err.Error(), string(dorisCompatModeInnerProductDuplicate)) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestResolveCompatMode_AutoPrefersInnerProductDuplicate(t *testing.T) { + repo, mock, _, cleanup := newTestRepo(t) + defer cleanup() + repo.compatModeRequested = dorisCompatModeAuto + + mock.ExpectQuery(`SELECT TABLE_NAME FROM information_schema.tables`). + WithArgs("weknora", "weknora_embeddings\\_%"). + WillReturnRows(sqlmock.NewRows([]string{"TABLE_NAME"})) + mock.ExpectQuery(`SELECT inner_product_approximate\(\[1.0\],\[1.0\]\)`). + WillReturnRows(sqlmock.NewRows([]string{"v"}).AddRow(1.0)) + mock.ExpectQuery(`SELECT cosine_distance_approximate\(\[1.0\],\[1.0\]\)`). + WillReturnError(errors.New("unsupported")) + + mode, err := repo.resolveCompatMode(context.Background()) + require.NoError(t, err) + assert.Equal(t, dorisCompatModeInnerProductDuplicate, mode) + assert.NoError(t, mock.ExpectationsWereMet()) +} + // --------------------------------------------------------------------------- // Engine wiring // --------------------------------------------------------------------------- diff --git a/internal/application/repository/retriever/doris/schema.go b/internal/application/repository/retriever/doris/schema.go index c599914a..1b60a66c 100644 --- a/internal/application/repository/retriever/doris/schema.go +++ b/internal/application/repository/retriever/doris/schema.go @@ -38,6 +38,10 @@ func (r *dorisRepository) ensureTable(ctx context.Context, dimension int) error if _, ok := r.initializedTables.Load(dimension); ok { return nil } + compatMode, err := r.resolveCompatMode(ctx) + if err != nil { + return err + } log := logger.GetLogger(ctx) tableName := r.getTableName(dimension) @@ -49,8 +53,8 @@ func (r *dorisRepository) ensureTable(ctx context.Context, dimension int) error } if !exists { - log.Infof("[Doris] Creating table %s with dimension %d", tableName, dimension) - if err := r.createTable(ctx, tableName, dimension); err != nil { + log.Infof("[Doris] Creating table %s with dimension %d in compat mode %s", tableName, dimension, compatMode) + if err := r.createTable(ctx, tableName, dimension, compatMode); err != nil { log.Errorf("[Doris] Failed to create table: %v", err) return fmt.Errorf("create table: %w", err) } @@ -93,7 +97,7 @@ func (r *dorisRepository) tableExists(ctx context.Context, tableName string) (bo // createTable 发出 CREATE TABLE DDL。Doris DDL 是同步的(除 ANN 索引构建外), // 返回成功即代表表已可写。 -func (r *dorisRepository) createTable(ctx context.Context, tableName string, dimension int) error { +func (r *dorisRepository) createTable(ctx context.Context, tableName string, dimension int, compatMode dorisCompatMode) error { buckets := r.bucketsNum if buckets <= 0 { buckets = defaultBucketsNum @@ -103,22 +107,42 @@ func (r *dorisRepository) createTable(ctx context.Context, tableName string, dim replication = defaultReplicationNum } - ddl := buildCreateTableDDL(tableName, dimension, buckets, replication) + ddl := buildCreateTableDDL(tableName, dimension, buckets, replication, compatMode) _, err := r.db.ExecContext(ctx, ddl) + if err != nil && compatMode == dorisCompatModeLegacy { + return fmt.Errorf( + "legacy Doris table creation failed: %w. If your Doris build rejects ANN indexes on UNIQUE KEY tables, set %s=%s before creating embedding tables. %s is not interchangeable after %s_* tables are created", + err, + envDorisCompatMode, + dorisCompatModeInnerProductDuplicate, + envDorisCompatMode, + r.tableBaseName, + ) + } return err } // buildCreateTableDDL 根据维度生成 CREATE TABLE DDL。 // // 关键点: -// - UNIQUE KEY(id) + enable_unique_key_merge_on_write=true: -// 这是 partial update(Stream Load) 起作用的前提。 +// - DUPLICATE KEY(id):兼容当前 Doris/SelectDB 对 ANN 索引的表模型要求。 +// WeKnora 在 Go 端用 delete + insert 保持按 id 替换的写入语义。 // - INVERTED 索引覆盖所有过滤字段 + 中文分词的 content 全文索引。 -// - ANN 索引使用 HNSW + cosine_distance,与 Qdrant/Weaviate 的 cosine 相似度方向一致。 +// - ANN 索引使用 HNSW + inner_product;Doris 写入/查询前会对向量单位化, +// 因此整体仍保持与其他向量库一致的 cosine 相似度语义。 // // 注意:DDL 中 dimension / buckets / replication 三个数值字段是 Go 端格式化拼接的, // 不存在 SQL 注入风险(来源都是受控的 IndexConfig int)。 -func buildCreateTableDDL(tableName string, dimension, buckets, replication int) string { +func buildCreateTableDDL(tableName string, dimension, buckets, replication int, compatMode dorisCompatMode) string { + metricType := "inner_product" + keyMode := "DUPLICATE KEY(id)" + properties := fmt.Sprintf("\t\"replication_num\"=\"%d\"", replication) + if compatMode == dorisCompatModeLegacy { + metricType = "cosine_distance" + keyMode = "UNIQUE KEY(id)" + properties = fmt.Sprintf("\t\"replication_num\"=\"%d\",\n\t\"enable_unique_key_merge_on_write\"=\"true\"", replication) + } + const tpl = `CREATE TABLE IF NOT EXISTS ` + "`%s`" + ` ( id VARCHAR(64) NOT NULL, chunk_id VARCHAR(64), @@ -139,19 +163,18 @@ func buildCreateTableDDL(tableName string, dimension, buckets, replication int) INDEX idx_content (content) USING INVERTED PROPERTIES("parser"="chinese","support_phrase"="true"), INDEX idx_emb (embedding) USING ANN PROPERTIES( "index_type"="hnsw", - "metric_type"="cosine_distance", + "metric_type"="%s", "dim"="%d", "max_degree"="32", "ef_construction"="200" ) ) ENGINE=OLAP -UNIQUE KEY(id) +%s DISTRIBUTED BY HASH(id) BUCKETS %d PROPERTIES( - "replication_num"="%d", - "enable_unique_key_merge_on_write"="true" + %s );` - return fmt.Sprintf(tpl, tableName, dimension, buckets, replication) + return fmt.Sprintf(tpl, tableName, metricType, dimension, keyMode, buckets, properties) } // waitANNReady 轮询 SHOW INDEX,等待 ANN 索引进入 FINISHED 状态。 diff --git a/internal/application/repository/retriever/doris/streamload.go b/internal/application/repository/retriever/doris/streamload.go index b6c056d1..a1ca4176 100644 --- a/internal/application/repository/retriever/doris/streamload.go +++ b/internal/application/repository/retriever/doris/streamload.go @@ -200,30 +200,117 @@ func chunkRows(rows []map[string]any, maxBytes int) [][]map[string]any { // --------------------------------------------------------------------------- // BatchUpdateChunkEnabledStatus 批量更新 chunk 的 is_enabled 字段。 -// -// 由于 Stream Load partial update 需要主键列 id,但 chunk_id 不一定等于 id, -// 这里做一个查表步骤:通过 chunk_id IN (...) 查出对应的 (id, table) 列表, -// 再分表用 partial update 写回。 -// -// 时间复杂度上是 O(N):扫描一遍 chunk_id IN (...) 在 INVERTED 索引下走点查。 +// legacy 模式走 Stream Load partial update;inner_product_duplicate 模式改为读整行后 replaceRows 写回。 func (r *dorisRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool, ) error { if len(chunkStatusMap) == 0 { return nil } + compatMode, err := r.resolveCompatMode(ctx) + if err != nil { + return err + } + if !compatMode.usesRewriteChunkUpdates() { + return r.batchUpdateChunkEnabledStatusLegacy(ctx, chunkStatusMap) + } chunkIDs := make([]string, 0, len(chunkStatusMap)) for id := range chunkStatusMap { chunkIDs = append(chunkIDs, id) } + return r.rewriteChunkRows(ctx, chunkIDs, func(row *DorisVectorEmbedding) bool { + enabled, ok := chunkStatusMap[row.ChunkID] + if !ok || row.IsEnabled == enabled { + return false + } + row.IsEnabled = enabled + return true + }, "rewrite is_enabled") +} + +// BatchUpdateChunkTagID 批量更新 chunk 的 tag_id 字段。逻辑与 EnabledStatus 一致。 +func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context, + chunkTagMap map[string]string, +) error { + if len(chunkTagMap) == 0 { + return nil + } + compatMode, err := r.resolveCompatMode(ctx) + if err != nil { + return err + } + if !compatMode.usesRewriteChunkUpdates() { + return r.batchUpdateChunkTagIDLegacy(ctx, chunkTagMap) + } + + chunkIDs := make([]string, 0, len(chunkTagMap)) + for id := range chunkTagMap { + chunkIDs = append(chunkIDs, id) + } + + return r.rewriteChunkRows(ctx, chunkIDs, func(row *DorisVectorEmbedding) bool { + tagID, ok := chunkTagMap[row.ChunkID] + if !ok || row.TagID == tagID { + return false + } + row.TagID = tagID + return true + }, "rewrite tag_id") +} + +func (r *dorisRepository) rewriteChunkRows(ctx context.Context, + chunkIDs []string, + mutate func(*DorisVectorEmbedding) bool, + action string, +) error { + if len(chunkIDs) == 0 { + return nil + } + + tables, err := r.listEmbeddingTables(ctx) + if err != nil { + return fmt.Errorf("list tables: %w", err) + } + + for _, table := range tables { + rows, err := r.loadRowsByChunkIDs(ctx, table, chunkIDs) + if err != nil { + return fmt.Errorf("load chunk rows from %s: %w", table, err) + } + + updated := make([]*DorisVectorEmbedding, 0, len(rows)) + for _, row := range rows { + if !mutate(row) { + continue + } + updated = append(updated, row) + } + if len(updated) == 0 { + continue + } + + if err := r.replaceRows(ctx, table, updated); err != nil { + return fmt.Errorf("%s in %s: %w", action, table, err) + } + } + return nil +} + +func (r *dorisRepository) batchUpdateChunkEnabledStatusLegacy(ctx context.Context, + chunkStatusMap map[string]bool, +) error { + chunkIDs := make([]string, 0, len(chunkStatusMap)) + for id := range chunkStatusMap { + chunkIDs = append(chunkIDs, id) + } + mapping, err := r.lookupChunkRowKeys(ctx, chunkIDs) if err != nil { return err } - // 按表分组 -> 每行 (id, is_enabled)。 byTable := make(map[string][]map[string]any) for chunkID, locations := range mapping { enabled, ok := chunkStatusMap[chunkID] @@ -245,14 +332,9 @@ func (r *dorisRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, return nil } -// BatchUpdateChunkTagID 批量更新 chunk 的 tag_id 字段。逻辑与 EnabledStatus 一致。 -func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context, +func (r *dorisRepository) batchUpdateChunkTagIDLegacy(ctx context.Context, chunkTagMap map[string]string, ) error { - if len(chunkTagMap) == 0 { - return nil - } - chunkIDs := make([]string, 0, len(chunkTagMap)) for id := range chunkTagMap { chunkIDs = append(chunkIDs, id) @@ -284,6 +366,39 @@ func (r *dorisRepository) BatchUpdateChunkTagID(ctx context.Context, return nil } +func (r *dorisRepository) loadRowsByChunkIDs(ctx context.Context, + table string, chunkIDs []string, +) ([]*DorisVectorEmbedding, error) { + if len(chunkIDs) == 0 { + return nil, nil + } + + placeholders := make([]string, len(chunkIDs)) + args := make([]any, len(chunkIDs)) + for i, v := range chunkIDs { + placeholders[i] = "?" + args[i] = v + } + + stmt := fmt.Sprintf( + "SELECT %s FROM `%s` WHERE %s IN (%s)", + strings.Join(columnsForCopy, ", "), + table, + fieldChunkID, + strings.Join(placeholders, ", "), + ) + rows, err := r.db.QueryContext(ctx, stmt, args...) + if err != nil { + return nil, err + } + batch, err := scanCopyRows(rows) + _ = rows.Close() + if err != nil { + return nil, err + } + return batch, nil +} + // rowLocation 表示某行在哪个表里、主键 id 是什么。 type rowLocation struct { table string diff --git a/internal/application/repository/retriever/doris/structs.go b/internal/application/repository/retriever/doris/structs.go index f7a78bfa..96a4ffd5 100644 --- a/internal/application/repository/retriever/doris/structs.go +++ b/internal/application/repository/retriever/doris/structs.go @@ -10,10 +10,13 @@ import ( // // 通信通道: // - 读写主链路:MySQL 协议(database/sql + go-sql-driver/mysql),FE 默认 9030 端口。 -// - Stream Load:HTTP(FE 默认 8030 端口),用于 BatchUpdate* 的 partial update。 +// - Stream Load:HTTP(FE 默认 8030 端口),legacy 模式用于 partial update。 // -// 表结构按维度分表:_,UNIQUE KEY(id), -// 同时建立倒排索引(filter / 全文)+ ANN(HNSW) 索引。 +// 表结构按维度分表:_。 +// 兼容模式由 DORIS_COMPAT_MODE 决定: +// - legacy:UNIQUE KEY(id) + cosine_distance ANN + Stream Load partial update +// - inner_product_duplicate:DUPLICATE KEY(id) + normalized inner product + delete/insert rewrite +// 该设置在 embedding 表创建后不可直接互换;切换模式前需要重建这些表。 // // 与 Qdrant/Milvus/Weaviate 一样,initializedTables 缓存"已确保存在"的维度, // 避免每次写入都打 SHOW TABLES。 @@ -32,6 +35,10 @@ type dorisRepository struct { tableBaseName string bucketsNum int // 0 -> default 10 replicationNum int // 0 -> default 1 + compatModeRequested dorisCompatMode + compatModeResolved dorisCompatMode + compatResolveOnce sync.Once + compatResolveErr error // 已经确保过 ensureTable 的维度集合:dim -> true。 initializedTables sync.Map @@ -55,8 +62,7 @@ type DorisVectorEmbedding struct { } // DorisVectorEmbeddingWithScore 是检索结果的领域模型, -// Score 在向量检索时存储 (1 - cosine_distance_approximate), -// 在关键词检索时统一赋 1.0(与 Qdrant 行为一致)。 +// Score 在向量检索时按当前 compat mode 计算,在关键词检索时统一赋 1.0。 type DorisVectorEmbeddingWithScore struct { DorisVectorEmbedding Score float64