feat: support milvus vectordb for knowledge retrieve
Signed-off-by: joeyczheng <joeyczheng@tencent.com>
This commit is contained in:
+17
-1
@@ -24,7 +24,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
# 主数据库类型(postgres/mysql)
|
||||
DB_DRIVER=postgres
|
||||
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant/milvus)
|
||||
RETRIEVE_DRIVER=postgres
|
||||
|
||||
# 文件存储类型(local/minio/cos/tos)
|
||||
@@ -226,3 +226,19 @@ WEKNORA_SANDBOX_DOCKER_IMAGE=wechatopenai/weknora-sandbox:latest
|
||||
|
||||
# APK 镜像源设置(可选)
|
||||
APK_MIRROR_ARG=mirrors.tencent.com
|
||||
|
||||
# 如果使用Milvus作为向量存储,需要配置以下参数
|
||||
# Milvus服务地址
|
||||
# MILVUS_ADDRESS=milvus:19530
|
||||
|
||||
# Milvus集合名称,用于存储向量数据
|
||||
# MILVUS_COLLECTION=weknora_embeddings
|
||||
|
||||
# Milvus 用户名(可选)
|
||||
# MILVUS_USERNAME=your_milvus_username
|
||||
|
||||
# Milvus 密码(可选)
|
||||
# MILVUS_PASSWORD=your_milvus_password
|
||||
|
||||
# Milvus 数据库名称(可选)
|
||||
# MILVUS_DB_NAME=your_milvus_db_name
|
||||
|
||||
@@ -219,12 +219,13 @@ deps:
|
||||
go mod download
|
||||
|
||||
# Build for production
|
||||
# google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn for qdrant milvus proto conflict
|
||||
build-prod:
|
||||
VERSION=$$(git describe --tags --abbrev=0 2>/dev/null || echo "$${VERSION:-unknown}"); \
|
||||
COMMIT_ID=$${COMMIT_ID:-unknown}; \
|
||||
BUILD_TIME=$${BUILD_TIME:-unknown}; \
|
||||
GO_VERSION=$${GO_VERSION:-unknown}; \
|
||||
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION'"; \
|
||||
LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION' -X 'google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=warn'"; \
|
||||
go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH)
|
||||
|
||||
download_spatial:
|
||||
|
||||
@@ -81,6 +81,8 @@ services:
|
||||
- QDRANT_COLLECTION=${QDRANT_COLLECTION:-weknora_embeddings}
|
||||
- QDRANT_API_KEY=${QDRANT_API_KEY:-}
|
||||
- QDRANT_USE_TLS=${QDRANT_USE_TLS:-false}
|
||||
- MILVUS_ADDRESS=milvus:19530
|
||||
- MILVUS_COLLECTION=${MILVUS_COLLECTION:-weknora_embeddings}
|
||||
- DOCREADER_ADDR=docreader:50051
|
||||
- STORAGE_TYPE=${STORAGE_TYPE:-}
|
||||
- LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-}
|
||||
@@ -275,6 +277,34 @@ services:
|
||||
- qdrant
|
||||
- full
|
||||
|
||||
milvus:
|
||||
image: milvusdb/milvus:v2.6.11
|
||||
container_name: WeKnora-milvus
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
command: ["milvus", "run", "standalone"]
|
||||
environment:
|
||||
- ETCD_USE_EMBED=true
|
||||
- ETCD_DATA_DIR=/var/lib/milvus/etcd
|
||||
- COMMON_STORAGETYPE=local
|
||||
- DEPLOY_MODE=STANDALONE
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
||||
interval: 30s
|
||||
start_period: 90s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
ports:
|
||||
- "19530:19530"
|
||||
- "9091:9091"
|
||||
volumes:
|
||||
- milvus_data:/var/lib/milvus
|
||||
networks:
|
||||
- WeKnora-network
|
||||
restart: unless-stopped
|
||||
profiles:
|
||||
- milvus
|
||||
|
||||
networks:
|
||||
WeKnora-network:
|
||||
driver: bridge
|
||||
@@ -286,3 +316,4 @@ volumes:
|
||||
minio_data:
|
||||
neo4j-data:
|
||||
qdrant_data:
|
||||
milvus_data:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
module github.com/Tencent/WeKnora
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.2
|
||||
go 1.24.11
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
@@ -19,10 +17,11 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hibiken/asynq v0.25.1
|
||||
github.com/mark3labs/mcp-go v0.43.0
|
||||
github.com/milvus-io/milvus/client/v2 v2.6.2
|
||||
github.com/minio/minio-go/v7 v7.0.90
|
||||
github.com/neo4j/neo4j-go-driver/v6 v6.0.0-alpha.1
|
||||
github.com/ollama/ollama v0.11.4
|
||||
github.com/panjf2000/ants/v2 v2.11.2
|
||||
github.com/panjf2000/ants/v2 v2.11.3
|
||||
github.com/parquet-go/parquet-go v0.25.0
|
||||
github.com/pganalyze/pg_query_go/v6 v6.1.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
@@ -60,23 +59,32 @@ require (
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.4.1 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/blang/semver/v4 v4.0.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/cilium/ebpf v0.11.0 // indirect
|
||||
github.com/clbanning/mxj v1.8.4 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/cockroachdb/errors v1.9.1 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
|
||||
github.com/cockroachdb/redact v1.1.3 // indirect
|
||||
github.com/containerd/cgroups/v3 v3.0.3 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect
|
||||
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect
|
||||
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.24 // indirect
|
||||
@@ -88,17 +96,21 @@ require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
|
||||
github.com/fsnotify/fsnotify v1.8.0 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/getsentry/sentry-go v0.12.0 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.20.2 // indirect
|
||||
github.com/go-openapi/spec v0.20.4 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
@@ -107,11 +119,19 @@ require (
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/godbus/dbus/v5 v5.0.4 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/btree v1.1.2 // indirect
|
||||
github.com/google/flatbuffers v25.9.23+incompatible // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.16.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
@@ -122,47 +142,90 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.6.8-0.20251223041313-25746c47c1a7 // indirect
|
||||
github.com/milvus-io/milvus/pkg/v2 v2.6.7-0.20251201120310-af64f2acba38 // indirect
|
||||
github.com/minio/crc64nvme v1.0.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/prometheus/client_golang v1.20.5 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.54.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.7.0 // indirect
|
||||
github.com/samber/lo v1.27.0 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/soheilhy/cmux v0.1.5 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/afero v1.12.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tidwall/gjson v1.17.1 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
|
||||
go.etcd.io/etcd/server/v3 v3.5.5 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/automaxprocs v1.5.3 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
@@ -174,7 +237,14 @@ require (
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
k8s.io/apimachinery v0.32.3 // indirect
|
||||
sigs.k8s.io/yaml v1.4.0 // indirect
|
||||
)
|
||||
|
||||
replace go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
package milvus
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// operatorAnd is the "and" operator.
|
||||
operatorAnd = "and"
|
||||
|
||||
// operatorOr is the "or" operator.
|
||||
operatorOr = "or"
|
||||
|
||||
// operatorEqual is the "equal" operator.
|
||||
operatorEqual = "eq"
|
||||
|
||||
// operatorNotEqual is the "not equal" operator.
|
||||
operatorNotEqual = "ne"
|
||||
|
||||
// operatorGreaterThan is the "greater than" operator.
|
||||
operatorGreaterThan = "gt"
|
||||
|
||||
// operatorGreaterThanOrEqual is the "greater than or equal" operator.
|
||||
operatorGreaterThanOrEqual = "gte"
|
||||
|
||||
// operatorLessThan is the "less than" operator.
|
||||
operatorLessThan = "lt"
|
||||
|
||||
// operatorLessThanOrEqual is the "less than or equal" operator.
|
||||
operatorLessThanOrEqual = "lte"
|
||||
|
||||
// operatorIn is the "in" operator.
|
||||
operatorIn = "in"
|
||||
|
||||
// operatorNotIn is the "not in" operator.
|
||||
operatorNotIn = "not in"
|
||||
|
||||
// operatorLike is the "contains" operator.
|
||||
operatorLike = "like"
|
||||
|
||||
// operatorNotLike is the "not contains" operator.
|
||||
operatorNotLike = "not like"
|
||||
|
||||
// operatorBetween is the "between" operator.
|
||||
operatorBetween = "between"
|
||||
)
|
||||
|
||||
var comparisonOperators = map[string]string{
|
||||
operatorEqual: "==",
|
||||
operatorNotEqual: "!=",
|
||||
operatorGreaterThan: ">",
|
||||
operatorGreaterThanOrEqual: ">=",
|
||||
operatorLessThan: "<",
|
||||
operatorLessThanOrEqual: "<=",
|
||||
operatorLike: "like",
|
||||
operatorNotLike: "not like",
|
||||
}
|
||||
|
||||
type convertResult struct {
|
||||
exprStr string
|
||||
params map[string]any
|
||||
}
|
||||
|
||||
type filter struct{}
|
||||
|
||||
func (c *filter) Convert(cond *universalFilterCondition) (*convertResult, error) {
|
||||
var counter int
|
||||
return c.convertCondition(cond, &counter)
|
||||
}
|
||||
|
||||
func (c *filter) convertComparisonCondition(
|
||||
cond *universalFilterCondition,
|
||||
counter *int,
|
||||
) (*convertResult, error) {
|
||||
condField := cond.Field
|
||||
if condField == "" || cond.Value == nil {
|
||||
return nil, fmt.Errorf("milvus filter condition is nil")
|
||||
}
|
||||
operator, ok := comparisonOperators[cond.Operator]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported comparison operator: %s", cond.Operator)
|
||||
}
|
||||
|
||||
paramName := c.convertParamName(cond.Field, counter)
|
||||
return &convertResult{
|
||||
exprStr: fmt.Sprintf("%s %s {%s}", condField, operator, paramName),
|
||||
params: map[string]any{paramName: cond.Value},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *filter) convertLogicalCondition(
|
||||
cond *universalFilterCondition,
|
||||
counter *int,
|
||||
) (*convertResult, error) {
|
||||
if cond.Value == nil {
|
||||
return nil, fmt.Errorf("milvus filter condition is nil")
|
||||
}
|
||||
conds, ok := cond.Value.([]*universalFilterCondition)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid logical condition value type")
|
||||
}
|
||||
|
||||
var condResult *convertResult
|
||||
for _, childCond := range conds {
|
||||
childRes, err := c.convertCondition(childCond, counter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if childRes == nil || childRes.exprStr == "" {
|
||||
continue
|
||||
}
|
||||
if condResult == nil {
|
||||
condResult = childRes
|
||||
continue
|
||||
}
|
||||
|
||||
condResult.exprStr = fmt.Sprintf(
|
||||
"(%s) %s (%s)",
|
||||
condResult.exprStr,
|
||||
strings.ToLower(cond.Operator),
|
||||
childRes.exprStr,
|
||||
)
|
||||
maps.Copy(condResult.params, childRes.params)
|
||||
}
|
||||
|
||||
if condResult == nil {
|
||||
return nil, fmt.Errorf("empty logical condition")
|
||||
}
|
||||
return condResult, nil
|
||||
}
|
||||
|
||||
func (c *filter) convertCondition(
|
||||
cond *universalFilterCondition,
|
||||
counter *int,
|
||||
) (*convertResult, error) {
|
||||
if cond == nil {
|
||||
return nil, fmt.Errorf("milvus filter condition is nil")
|
||||
}
|
||||
switch cond.Operator {
|
||||
case operatorEqual, operatorNotEqual, operatorGreaterThan,
|
||||
operatorGreaterThanOrEqual, operatorLessThan,
|
||||
operatorLessThanOrEqual, operatorLike, operatorNotLike:
|
||||
return c.convertComparisonCondition(cond, counter)
|
||||
case operatorAnd, operatorOr:
|
||||
return c.convertLogicalCondition(cond, counter)
|
||||
case operatorIn, operatorNotIn:
|
||||
return c.convertInCondition(cond, counter)
|
||||
case operatorBetween:
|
||||
return c.convertBetweenCondition(cond, counter)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported operator: %v", cond.Operator)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *filter) convertInCondition(
|
||||
cond *universalFilterCondition,
|
||||
counter *int,
|
||||
) (*convertResult, error) {
|
||||
condField := cond.Field
|
||||
if condField == "" || cond.Value == nil {
|
||||
return nil, fmt.Errorf("milvus filter condition is nil")
|
||||
}
|
||||
|
||||
s := reflect.ValueOf(cond.Value)
|
||||
if s.Kind() != reflect.Slice || s.Len() <= 0 {
|
||||
return nil, fmt.Errorf("in operator value must be a slice with at least one value: %v", cond.Value)
|
||||
}
|
||||
|
||||
paramName := c.convertParamName(cond.Field, counter)
|
||||
return &convertResult{
|
||||
exprStr: fmt.Sprintf("%s %s {%s}", condField, strings.ToLower(cond.Operator), paramName),
|
||||
params: map[string]any{paramName: cond.Value},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *filter) convertBetweenCondition(
|
||||
cond *universalFilterCondition,
|
||||
counter *int,
|
||||
) (*convertResult, error) {
|
||||
condField := cond.Field
|
||||
if condField == "" || cond.Value == nil {
|
||||
return nil, fmt.Errorf("milvus filter condition is nil")
|
||||
}
|
||||
|
||||
value := reflect.ValueOf(cond.Value)
|
||||
if value.Kind() != reflect.Slice || value.Len() != 2 {
|
||||
return nil, fmt.Errorf("between operator value must be a slice with two elements: %v", cond.Value)
|
||||
}
|
||||
|
||||
paramBase := c.convertParamName(cond.Field, counter)
|
||||
paramName1 := fmt.Sprintf("%s_%d", paramBase, 0)
|
||||
paramName2 := fmt.Sprintf("%s_%d", paramBase, 1)
|
||||
return &convertResult{
|
||||
exprStr: fmt.Sprintf("%s >= {%s} and %s <= {%s}", condField, paramName1, condField, paramName2),
|
||||
params: map[string]any{
|
||||
paramName1: value.Index(0).Interface(),
|
||||
paramName2: value.Index(1).Interface(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func formatValue(value any) string {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return fmt.Sprintf("\"%s\"", escapeDoubleQuotes(v))
|
||||
case int, int8, int16, int32, int64:
|
||||
return fmt.Sprintf("%d", v)
|
||||
case uint, uint8, uint16, uint32, uint64:
|
||||
return fmt.Sprintf("%d", v)
|
||||
case float32, float64:
|
||||
return fmt.Sprintf("%v", v)
|
||||
case bool:
|
||||
if v {
|
||||
return "true"
|
||||
}
|
||||
return "false"
|
||||
case time.Time:
|
||||
return fmt.Sprintf("%d", v.Unix())
|
||||
default:
|
||||
return fmt.Sprintf("\"%v\"", value)
|
||||
}
|
||||
}
|
||||
|
||||
// escapeDoubleQuotes escapes double quotes in a string for use in Milvus expressions.
|
||||
func escapeDoubleQuotes(s string) string {
|
||||
return strings.ReplaceAll(s, "\"", "\\\"")
|
||||
}
|
||||
|
||||
// convertParamName converts field name to a valid Milvus template parameter name.
|
||||
// Milvus template parameters don't support '.' character, so we replace it with '_'.
|
||||
func (c *filter) convertParamName(field string, counter *int) string {
|
||||
*counter++
|
||||
return fmt.Sprintf("%s_%d", strings.ReplaceAll(field, ".", "_"), *counter)
|
||||
}
|
||||
|
||||
type universalFilterCondition struct {
|
||||
Field string `json:"field,omitempty" jsonschema:"description=The metadata field to filter on (required for comparison operators)"`
|
||||
Operator string `json:"operator" jsonschema:"description=The operator to use,enum=eq,enum=ne,enum=gt,enum=gte,enum=lt,enum=lte,enum=in,enum=not in,enum=like,enum=not like,enum=between,enum=and,enum=or"`
|
||||
Value any `json:"value,omitempty" jsonschema:"description=The value to compare against (for comparison operators) or array of sub-conditions (for logical operators and/or)"`
|
||||
}
|
||||
|
||||
func (c *universalFilterCondition) UnmarshalJSON(data []byte) error {
|
||||
type Alias struct {
|
||||
Field string `json:"field,omitempty"`
|
||||
Operator string `json:"operator"`
|
||||
Value any `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
var aux Alias
|
||||
if err := json.Unmarshal(data, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.Field = aux.Field
|
||||
c.Operator = strings.ToLower(aux.Operator)
|
||||
|
||||
// Handle logical operators (and/or) - Value should be []*UniversalFilterCondition
|
||||
if c.Operator == operatorAnd || c.Operator == operatorOr {
|
||||
// Value can be an array of conditions
|
||||
valueSlice, ok := aux.Value.([]any)
|
||||
if !ok {
|
||||
return fmt.Errorf("logical operator %s requires an array of conditions", c.Operator)
|
||||
}
|
||||
|
||||
conditions := make([]*universalFilterCondition, 0, len(valueSlice))
|
||||
for i, v := range valueSlice {
|
||||
condBytes, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal condition at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
var cond universalFilterCondition
|
||||
if err := json.Unmarshal(condBytes, &cond); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal condition at index %d: %w", i, err)
|
||||
}
|
||||
conditions = append(conditions, &cond)
|
||||
}
|
||||
c.Value = conditions
|
||||
} else {
|
||||
c.Value = aux.Value
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON implements custom JSON marshaling for UniversalFilterCondition.
|
||||
func (c *universalFilterCondition) MarshalJSON() ([]byte, error) {
|
||||
type Alias struct {
|
||||
Field string `json:"field,omitempty"`
|
||||
Operator string `json:"operator"`
|
||||
Value any `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
return json.Marshal(&Alias{
|
||||
Field: c.Field,
|
||||
Operator: c.Operator,
|
||||
Value: c.Value,
|
||||
})
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
package milvus
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
client "github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
)
|
||||
|
||||
type milvusRepository struct {
|
||||
filter
|
||||
client *client.Client
|
||||
collectionBaseName string
|
||||
// Cache for initialized collections (dimension -> true)
|
||||
initializedCollections sync.Map
|
||||
}
|
||||
|
||||
type MilvusVectorEmbedding struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceType int `json:"source_type"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
TagID string `json:"tag_id"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
}
|
||||
|
||||
type MilvusVectorEmbeddingWithScore struct {
|
||||
MilvusVectorEmbedding
|
||||
Score float64
|
||||
}
|
||||
@@ -17,11 +17,13 @@ import (
|
||||
_ "github.com/duckdb/duckdb-go/v2"
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/dig"
|
||||
"google.golang.org/grpc"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
|
||||
@@ -30,6 +32,7 @@ import (
|
||||
memoryRepo "github.com/Tencent/WeKnora/internal/application/repository/memory/neo4j"
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
milvusRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/milvus"
|
||||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
@@ -562,6 +565,43 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(retrieveDriver, "milvus") {
|
||||
milvusCfg := milvusclient.ClientConfig{
|
||||
DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)},
|
||||
}
|
||||
milvusAddress := os.Getenv("MILVUS_ADDRESS")
|
||||
if milvusAddress == "" {
|
||||
milvusAddress = "localhost:19530"
|
||||
}
|
||||
milvusCfg.Address = milvusAddress
|
||||
milvusUsername := os.Getenv("MILVUS_USERNAME")
|
||||
if milvusUsername != "" {
|
||||
milvusCfg.Username = milvusUsername
|
||||
}
|
||||
milvusPassword := os.Getenv("MILVUS_PASSWORD")
|
||||
if milvusPassword != "" {
|
||||
milvusCfg.Password = milvusPassword
|
||||
}
|
||||
milvusDBName := os.Getenv("MILVUS_DB_NAME")
|
||||
if milvusDBName != "" {
|
||||
milvusCfg.DBName = milvusDBName
|
||||
}
|
||||
milvusCli, err := milvusclient.New(context.Background(), &milvusCfg)
|
||||
if err != nil {
|
||||
log.Errorf("Create milvus client failed: %v", err)
|
||||
} else {
|
||||
milvusRepository := milvusRepo.NewMilvusRetrieveEngineRepository(milvusCli)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(
|
||||
milvusRepository, types.MilvusRetrieverEngineType,
|
||||
),
|
||||
); err != nil {
|
||||
log.Errorf("Register milvus retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register milvus retrieve engine success")
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ const (
|
||||
InfinityRetrieverEngineType RetrieverEngineType = "infinity"
|
||||
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
|
||||
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
|
||||
MilvusRetrieverEngineType RetrieverEngineType = "milvus"
|
||||
)
|
||||
|
||||
// RetrieverType represents the type of retriever
|
||||
|
||||
@@ -27,6 +27,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType},
|
||||
},
|
||||
"milvus": {
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
|
||||
},
|
||||
}
|
||||
|
||||
// GetRetrieverEngineMapping returns the retriever engine mapping
|
||||
|
||||
Reference in New Issue
Block a user