feat: support milvus vectordb for knowledge retrieve

Signed-off-by: joeyczheng <joeyczheng@tencent.com>
This commit is contained in:
joeyczheng
2026-02-26 21:48:38 +08:00
committed by lyingbug
parent daf7dee4d2
commit 1b56e99cac
11 changed files with 4725 additions and 22 deletions
+17 -1
View File
@@ -24,7 +24,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# 主数据库类型(postgres/mysql)
DB_DRIVER=postgres
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant/milvus)
RETRIEVE_DRIVER=postgres
# 文件存储类型(local/minio/cos/tos)
@@ -226,3 +226,19 @@ WEKNORA_SANDBOX_DOCKER_IMAGE=wechatopenai/weknora-sandbox:latest
# APK 镜像源设置(可选)
APK_MIRROR_ARG=mirrors.tencent.com
# 如果使用Milvus作为向量存储,需要配置以下参数
# Milvus服务地址
# MILVUS_ADDRESS=milvus:19530
# Milvus集合名称,用于存储向量数据
# MILVUS_COLLECTION=weknora_embeddings
# Milvus 用户名(可选)
# MILVUS_USERNAME=your_milvus_username
# Milvus 密码(可选)
# MILVUS_PASSWORD=your_milvus_password
# Milvus 数据库名称(可选)
# MILVUS_DB_NAME=your_milvus_db_name
+2 -1
View File
@@ -219,12 +219,13 @@ deps:
go mod download
# Build for production
# google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn for qdrant milvus proto conflict
build-prod:
VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \
COMMIT_ID=$${COMMIT_ID:-unknown}; \
BUILD_TIME=$${BUILD_TIME:-unknown}; \
GO_VERSION=$${GO_VERSION:-unknown}; \
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION'"; \
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
download_spatial:
+31
View File
@@ -81,6 +81,8 @@ services:
- QDRANT_COLLECTION=${QDRANT_COLLECTION:-weknora_embeddings}
- QDRANT_API_KEY=${QDRANT_API_KEY:-}
- QDRANT_USE_TLS=${QDRANT_USE_TLS:-false}
- MILVUS_ADDRESS=milvus:19530
- MILVUS_COLLECTION=${MILVUS_COLLECTION:-weknora_embeddings}
- DOCREADER_ADDR=docreader:50051
- STORAGE_TYPE=${STORAGE_TYPE:-}
- LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-}
@@ -275,6 +277,34 @@ services:
- qdrant
- full
milvus:
image: milvusdb/milvus:v2.6.11
container_name: WeKnora-milvus
security_opt:
- seccomp:unconfined
command: ["milvus", "run", "standalone"]
environment:
- ETCD_USE_EMBED=true
- ETCD_DATA_DIR=/var/lib/milvus/etcd
- COMMON_STORAGETYPE=local
- DEPLOY_MODE=STANDALONE
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
interval: 30s
start_period: 90s
timeout: 20s
retries: 3
ports:
- "19530:19530"
- "9091:9091"
volumes:
- milvus_data:/var/lib/milvus
networks:
- WeKnora-network
restart: unless-stopped
profiles:
- milvus
networks:
WeKnora-network:
driver: bridge
@@ -286,3 +316,4 @@ volumes:
minio_data:
neo4j-data:
qdrant_data:
milvus_data:
+81 -11
View File
@@ -1,8 +1,6 @@
module github.com/Tencent/WeKnora
go 1.24.0
toolchain go1.24.2
go 1.24.11
require (
github.com/PuerkitoBio/goquery v1.10.3
@@ -19,10 +17,11 @@ require (
github.com/google/uuid v1.6.0
github.com/hibiken/asynq v0.25.1
github.com/mark3labs/mcp-go v0.43.0
github.com/milvus-io/milvus/client/v2 v2.6.2
github.com/minio/minio-go/v7 v7.0.90
github.com/neo4j/neo4j-go-driver/v6 v6.0.0-alpha.1
github.com/ollama/ollama v0.11.4
github.com/panjf2000/ants/v2 v2.11.2
github.com/panjf2000/ants/v2 v2.11.3
github.com/parquet-go/parquet-go v0.25.0
github.com/pganalyze/pg_query_go/v6 v6.1.0
github.com/pgvector/pgvector-go v0.3.0
@@ -60,23 +59,32 @@ require (
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/apache/arrow-go/v18 v18.4.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.2 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/cilium/ebpf v0.11.0 // indirect
github.com/clbanning/mxj v1.8.4 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/cockroachdb/errors v1.9.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.24 // indirect
@@ -88,17 +96,21 @@ require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/getsentry/sentry-go v0.12.0 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
@@ -107,11 +119,19 @@ require (
github.com/gobwas/ws v1.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/flatbuffers v25.9.23+incompatible // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/gax-go/v2 v2.16.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -122,47 +142,90 @@ require (
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.8-0.20251223041313-25746c47c1a7 // indirect
github.com/milvus-io/milvus/pkg/v2 v2.6.7-0.20251201120310-af64f2acba38 // indirect
github.com/minio/crc64nvme v1.0.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/prometheus/client_golang v1.20.5 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/quic-go/qpack v0.5.1 // indirect
github.com/quic-go/quic-go v0.54.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/samber/lo v1.27.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.12.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/twpayne/go-geom v1.6.1 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
go.etcd.io/etcd/client/v3 v3.5.5 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
go.etcd.io/etcd/server/v3 v3.5.5 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/automaxprocs v1.5.3 // indirect
go.uber.org/mock v0.5.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
golang.org/x/mod v0.31.0 // indirect
@@ -174,7 +237,14 @@ require (
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.40.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/apimachinery v0.32.3 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0
+3102 -9
View File
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,304 @@
package milvus
import (
"encoding/json"
"fmt"
"maps"
"reflect"
"strings"
"time"
)
const (
// operatorAnd is the "and" operator.
operatorAnd = "and"
// operatorOr is the "or" operator.
operatorOr = "or"
// operatorEqual is the "equal" operator.
operatorEqual = "eq"
// operatorNotEqual is the "not equal" operator.
operatorNotEqual = "ne"
// operatorGreaterThan is the "greater than" operator.
operatorGreaterThan = "gt"
// operatorGreaterThanOrEqual is the "greater than or equal" operator.
operatorGreaterThanOrEqual = "gte"
// operatorLessThan is the "less than" operator.
operatorLessThan = "lt"
// operatorLessThanOrEqual is the "less than or equal" operator.
operatorLessThanOrEqual = "lte"
// operatorIn is the "in" operator.
operatorIn = "in"
// operatorNotIn is the "not in" operator.
operatorNotIn = "not in"
// operatorLike is the "contains" operator.
operatorLike = "like"
// operatorNotLike is the "not contains" operator.
operatorNotLike = "not like"
// operatorBetween is the "between" operator.
operatorBetween = "between"
)
var comparisonOperators = map[string]string{
operatorEqual: "==",
operatorNotEqual: "!=",
operatorGreaterThan: ">",
operatorGreaterThanOrEqual: ">=",
operatorLessThan: "<",
operatorLessThanOrEqual: "<=",
operatorLike: "like",
operatorNotLike: "not like",
}
type convertResult struct {
exprStr string
params map[string]any
}
type filter struct{}
func (c *filter) Convert(cond *universalFilterCondition) (*convertResult, error) {
var counter int
return c.convertCondition(cond, &counter)
}
func (c *filter) convertComparisonCondition(
cond *universalFilterCondition,
counter *int,
) (*convertResult, error) {
condField := cond.Field
if condField == "" || cond.Value == nil {
return nil, fmt.Errorf("milvus filter condition is nil")
}
operator, ok := comparisonOperators[cond.Operator]
if !ok {
return nil, fmt.Errorf("unsupported comparison operator: %s", cond.Operator)
}
paramName := c.convertParamName(cond.Field, counter)
return &convertResult{
exprStr: fmt.Sprintf("%s %s {%s}", condField, operator, paramName),
params: map[string]any{paramName: cond.Value},
}, nil
}
func (c *filter) convertLogicalCondition(
cond *universalFilterCondition,
counter *int,
) (*convertResult, error) {
if cond.Value == nil {
return nil, fmt.Errorf("milvus filter condition is nil")
}
conds, ok := cond.Value.([]*universalFilterCondition)
if !ok {
return nil, fmt.Errorf("invalid logical condition value type")
}
var condResult *convertResult
for _, childCond := range conds {
childRes, err := c.convertCondition(childCond, counter)
if err != nil {
return nil, err
}
if childRes == nil || childRes.exprStr == "" {
continue
}
if condResult == nil {
condResult = childRes
continue
}
condResult.exprStr = fmt.Sprintf(
"(%s) %s (%s)",
condResult.exprStr,
strings.ToLower(cond.Operator),
childRes.exprStr,
)
maps.Copy(condResult.params, childRes.params)
}
if condResult == nil {
return nil, fmt.Errorf("empty logical condition")
}
return condResult, nil
}
func (c *filter) convertCondition(
cond *universalFilterCondition,
counter *int,
) (*convertResult, error) {
if cond == nil {
return nil, fmt.Errorf("milvus filter condition is nil")
}
switch cond.Operator {
case operatorEqual, operatorNotEqual, operatorGreaterThan,
operatorGreaterThanOrEqual, operatorLessThan,
operatorLessThanOrEqual, operatorLike, operatorNotLike:
return c.convertComparisonCondition(cond, counter)
case operatorAnd, operatorOr:
return c.convertLogicalCondition(cond, counter)
case operatorIn, operatorNotIn:
return c.convertInCondition(cond, counter)
case operatorBetween:
return c.convertBetweenCondition(cond, counter)
default:
return nil, fmt.Errorf("unsupported operator: %v", cond.Operator)
}
}
func (c *filter) convertInCondition(
cond *universalFilterCondition,
counter *int,
) (*convertResult, error) {
condField := cond.Field
if condField == "" || cond.Value == nil {
return nil, fmt.Errorf("milvus filter condition is nil")
}
s := reflect.ValueOf(cond.Value)
if s.Kind() != reflect.Slice || s.Len() <= 0 {
return nil, fmt.Errorf("in operator value must be a slice with at least one value: %v", cond.Value)
}
paramName := c.convertParamName(cond.Field, counter)
return &convertResult{
exprStr: fmt.Sprintf("%s %s {%s}", condField, strings.ToLower(cond.Operator), paramName),
params: map[string]any{paramName: cond.Value},
}, nil
}
func (c *filter) convertBetweenCondition(
cond *universalFilterCondition,
counter *int,
) (*convertResult, error) {
condField := cond.Field
if condField == "" || cond.Value == nil {
return nil, fmt.Errorf("milvus filter condition is nil")
}
value := reflect.ValueOf(cond.Value)
if value.Kind() != reflect.Slice || value.Len() != 2 {
return nil, fmt.Errorf("between operator value must be a slice with two elements: %v", cond.Value)
}
paramBase := c.convertParamName(cond.Field, counter)
paramName1 := fmt.Sprintf("%s_%d", paramBase, 0)
paramName2 := fmt.Sprintf("%s_%d", paramBase, 1)
return &convertResult{
exprStr: fmt.Sprintf("%s >= {%s} and %s <= {%s}", condField, paramName1, condField, paramName2),
params: map[string]any{
paramName1: value.Index(0).Interface(),
paramName2: value.Index(1).Interface(),
},
}, nil
}
func formatValue(value any) string {
switch v := value.(type) {
case string:
return fmt.Sprintf("\"%s\"", escapeDoubleQuotes(v))
case int, int8, int16, int32, int64:
return fmt.Sprintf("%d", v)
case uint, uint8, uint16, uint32, uint64:
return fmt.Sprintf("%d", v)
case float32, float64:
return fmt.Sprintf("%v", v)
case bool:
if v {
return "true"
}
return "false"
case time.Time:
return fmt.Sprintf("%d", v.Unix())
default:
return fmt.Sprintf("\"%v\"", value)
}
}
// escapeDoubleQuotes escapes double quotes in a string for use in Milvus expressions.
func escapeDoubleQuotes(s string) string {
return strings.ReplaceAll(s, "\"", "\\\"")
}
// convertParamName converts field name to a valid Milvus template parameter name.
// Milvus template parameters don't support '.' character, so we replace it with '_'.
func (c *filter) convertParamName(field string, counter *int) string {
*counter++
return fmt.Sprintf("%s_%d", strings.ReplaceAll(field, ".", "_"), *counter)
}
type universalFilterCondition struct {
Field string `json:"field,omitempty" jsonschema:"description=The metadata field to filter on (required for comparison operators)"`
Operator string `json:"operator" jsonschema:"description=The operator to use,enum=eq,enum=ne,enum=gt,enum=gte,enum=lt,enum=lte,enum=in,enum=not in,enum=like,enum=not like,enum=between,enum=and,enum=or"`
Value any `json:"value,omitempty" jsonschema:"description=The value to compare against (for comparison operators) or array of sub-conditions (for logical operators and/or)"`
}
func (c *universalFilterCondition) UnmarshalJSON(data []byte) error {
type Alias struct {
Field string `json:"field,omitempty"`
Operator string `json:"operator"`
Value any `json:"value,omitempty"`
}
var aux Alias
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
c.Field = aux.Field
c.Operator = strings.ToLower(aux.Operator)
// Handle logical operators (and/or) - Value should be []*UniversalFilterCondition
if c.Operator == operatorAnd || c.Operator == operatorOr {
// Value can be an array of conditions
valueSlice, ok := aux.Value.([]any)
if !ok {
return fmt.Errorf("logical operator %s requires an array of conditions", c.Operator)
}
conditions := make([]*universalFilterCondition, 0, len(valueSlice))
for i, v := range valueSlice {
condBytes, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("failed to marshal condition at index %d: %w", i, err)
}
var cond universalFilterCondition
if err := json.Unmarshal(condBytes, &cond); err != nil {
return fmt.Errorf("failed to unmarshal condition at index %d: %w", i, err)
}
conditions = append(conditions, &cond)
}
c.Value = conditions
} else {
c.Value = aux.Value
}
return nil
}
// MarshalJSON implements custom JSON marshaling for UniversalFilterCondition.
func (c *universalFilterCondition) MarshalJSON() ([]byte, error) {
type Alias struct {
Field string `json:"field,omitempty"`
Operator string `json:"operator"`
Value any `json:"value,omitempty"`
}
return json.Marshal(&Alias{
Field: c.Field,
Operator: c.Operator,
Value: c.Value,
})
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,33 @@
package milvus
import (
"sync"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
)
type milvusRepository struct {
filter
client *client.Client
collectionBaseName string
// Cache for initialized collections (dimension -> true)
initializedCollections sync.Map
}
type MilvusVectorEmbedding struct {
ID string `json:"id"`
Content string `json:"content"`
SourceID string `json:"source_id"`
SourceType int `json:"source_type"`
ChunkID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
TagID string `json:"tag_id"`
Embedding []float32 `json:"embedding"`
IsEnabled bool `json:"is_enabled"`
}
type MilvusVectorEmbeddingWithScore struct {
MilvusVectorEmbedding
Score float64
}
+40
View File
@@ -17,11 +17,13 @@ import (
_ "github.com/duckdb/duckdb-go/v2"
esv7 "github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v8"
"github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
"github.com/panjf2000/ants/v2"
"github.com/qdrant/go-client/qdrant"
"github.com/redis/go-redis/v9"
"go.uber.org/dig"
"google.golang.org/grpc"
"gorm.io/driver/postgres"
"gorm.io/gorm"
@@ -30,6 +32,7 @@ import (
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
milvusRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/milvus"
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
@@ -562,6 +565,43 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
}
}
}
if slices.Contains(retrieveDriver, "milvus") {
milvusCfg := milvusclient.ClientConfig{
DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)},
}
milvusAddress := os.Getenv("MILVUS_ADDRESS")
if milvusAddress == "" {
milvusAddress = "localhost:19530"
}
milvusCfg.Address = milvusAddress
milvusUsername := os.Getenv("MILVUS_USERNAME")
if milvusUsername != "" {
milvusCfg.Username = milvusUsername
}
milvusPassword := os.Getenv("MILVUS_PASSWORD")
if milvusPassword != "" {
milvusCfg.Password = milvusPassword
}
milvusDBName := os.Getenv("MILVUS_DB_NAME")
if milvusDBName != "" {
milvusCfg.DBName = milvusDBName
}
milvusCli, err := milvusclient.New(context.Background(), &milvusCfg)
if err != nil {
log.Errorf("Create milvus client failed: %v", err)
} else {
milvusRepository := milvusRepo.NewMilvusRetrieveEngineRepository(milvusCli)
if err := registry.Register(
retriever.NewKVHybridRetrieveEngine(
milvusRepository, types.MilvusRetrieverEngineType,
),
); err != nil {
log.Errorf("Register milvus retrieve engine failed: %v", err)
} else {
log.Infof("Register milvus retrieve engine success")
}
}
}
return registry, nil
}
+1
View File
@@ -10,6 +10,7 @@ const (
InfinityRetrieverEngineType RetrieverEngineType = "infinity"
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
MilvusRetrieverEngineType RetrieverEngineType = "milvus"
)
// RetrieverType represents the type of retriever
+4
View File
@@ -27,6 +27,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType},
{RetrieverType: VectorRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType},
},
"milvus": {
{RetrieverType: VectorRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
},
}
// GetRetrieverEngineMapping returns the retriever engine mapping