feat: Qdrant Vector Search support

Signed-off-by: Anush008 <anushshetty90@gmail.com>
This commit is contained in:
Anush008
2025-11-26 16:28:53 +05:30
committed by lyingbug
parent 3c3b457261
commit c7659c3d19
8 changed files with 725 additions and 58 deletions
+17 -1
View File
@@ -14,7 +14,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
# 主数据库类型(postgres/mysql)
DB_DRIVER=postgres
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8)
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
RETRIEVE_DRIVER=postgres
# 文件存储类型(local/minio/cos)
@@ -81,6 +81,22 @@ CONCURRENCY_POOL_SIZE=5
# ElasticSearch索引名称,用于存储向量数据
# ELASTICSEARCH_INDEX=WeKnora
# 如果使用Qdrant作为向量存储,需要配置以下参数
# Qdrant服务主机地址
# QDRANT_HOST=localhost
# Qdrant服务端口
# QDRANT_PORT=6334
# Qdrant集合名称,用于存储向量数据
# QDRANT_COLLECTION=weknora_embeddings
# Qdrant API密钥,如果需要身份验证(可选)
# QDRANT_API_KEY=your_qdrant_api_key
# 是否启用TLS加密连接(可选,默认为false)
# QDRANT_USE_TLS=false
# 如果使用MinIO作为文件存储,需要配置以下参数
# MinIO访问密钥
# MINIO_ACCESS_KEY_ID=your_minio_access_key
+16 -15
View File
@@ -19,6 +19,7 @@ require (
github.com/panjf2000/ants/v2 v2.11.2
github.com/parquet-go/parquet-go v0.25.0
github.com/pgvector/pgvector-go v0.3.0
github.com/qdrant/go-client v1.16.1
github.com/redis/go-redis/v9 v9.14.0
github.com/sashabaranov/go-openai v1.40.5
github.com/sirupsen/logrus v1.9.3
@@ -26,17 +27,17 @@ require (
github.com/stretchr/testify v1.11.1
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
github.com/yanyiwu/gojieba v1.4.5
go.opentelemetry.io/otel v1.37.0
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0
go.opentelemetry.io/otel/sdk v1.37.0
go.opentelemetry.io/otel/trace v1.37.0
go.opentelemetry.io/otel/trace v1.38.0
go.uber.org/dig v1.18.1
golang.org/x/crypto v0.42.0
golang.org/x/sync v0.17.0
google.golang.org/grpc v1.73.0
google.golang.org/protobuf v1.36.9
golang.org/x/crypto v0.44.0
golang.org/x/sync v0.18.0
google.golang.org/grpc v1.76.0
google.golang.org/protobuf v1.36.10
gorm.io/driver/postgres v1.5.11
gorm.io/gorm v1.25.12
)
@@ -72,7 +73,7 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
@@ -98,16 +99,16 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/metric v1.37.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.15.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/text v0.29.0 // indirect
golang.org/x/time v0.13.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+40 -36
View File
@@ -106,8 +106,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
@@ -159,6 +159,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/qdrant/go-client v1.16.1 h1:Jr47kz0k8I+U2sUm2UUO2eq2kL0fTcgjLPIz6a0RKuQ=
github.com/qdrant/go-client v1.16.1/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw=
github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE=
github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -166,8 +168,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
@@ -229,24 +231,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI=
github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgTNQVet8iTDW7oIk03tXHq+wkwIDnE=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/dig v1.18.1 h1:rLww6NuajVjeQn+49u5NcezUJEGwd5uXmyoCKW2g5Es=
@@ -257,31 +259,33 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc=
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:oDOGiMSXHL4sDTJvFvIB9nRQCGdLP1o/iVaqQK8zB+M=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -0,0 +1,572 @@
package qdrant
import (
"context"
"fmt"
"maps"
"os"
"slices"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
typesLocal "github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/qdrant/go-client/qdrant"
)
const (
envQdrantCollection = "QDRANT_COLLECTION"
defaultCollectionName = "weknora_embeddings"
fieldContent = "content"
fieldSourceID = "source_id"
fieldSourceType = "source_type"
fieldChunkID = "chunk_id"
fieldKnowledgeID = "knowledge_id"
fieldKnowledgeBaseID = "knowledge_base_id"
fieldEmbedding = "embedding"
)
// NewQdrantRetrieveEngineRepository creates and initializes a new Qdrant repository
func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.RetrieveEngineRepository {
log := logger.GetLogger(context.Background())
log.Info("[Qdrant] Initializing Qdrant retriever engine repository")
collectionName := os.Getenv(envQdrantCollection)
if collectionName == "" {
log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name")
collectionName = defaultCollectionName
}
res := &qdrantRepository{
client: client,
collectionName: collectionName,
}
log.Info("[Qdrant] Successfully initialized repository")
return res
}
func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType {
return typesLocal.QdrantRetrieverEngineType
}
func (q *qdrantRepository) Support() []typesLocal.RetrieverType {
return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType}
}
// EstimateStorageSize calculates the estimated storage size for a list of indices
func (q *qdrantRepository) EstimateStorageSize(ctx context.Context,
indexInfoList []*typesLocal.IndexInfo, params map[string]any,
) int64 {
var totalStorageSize int64
for _, embedding := range indexInfoList {
embeddingDB := toQdrantVectorEmbedding(embedding, params)
totalStorageSize += q.calculateStorageSize(embeddingDB)
}
logger.GetLogger(ctx).Infof(
"[Qdrant] Storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize,
)
return totalStorageSize
}
// Save stores a single point in Qdrant
func (q *qdrantRepository) Save(ctx context.Context,
embedding *typesLocal.IndexInfo,
additionalParams map[string]any,
) error {
log := logger.GetLogger(ctx)
log.Debugf("[Qdrant] Saving index for chunk ID: %s", embedding.ChunkID)
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
if len(embeddingDB.Embedding) == 0 {
err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID)
log.Errorf("[Qdrant] %v", err)
return err
}
pointID := uuid.New().String()
point := &qdrant.PointStruct{
Id: qdrant.NewID(pointID),
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
Payload: createPayload(embeddingDB),
}
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
CollectionName: q.collectionName,
Points: []*qdrant.PointStruct{point},
})
if err != nil {
log.Errorf("[Qdrant] Failed to save index: %v", err)
return err
}
log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID)
return nil
}
// BatchSave stores multiple points in Qdrant using batch upsert
func (q *qdrantRepository) BatchSave(ctx context.Context,
embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any,
) error {
log := logger.GetLogger(ctx)
if len(embeddingList) == 0 {
log.Warn("[Qdrant] Empty list provided to BatchSave, skipping")
return nil
}
log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList))
points := make([]*qdrant.PointStruct, 0, len(embeddingList))
for _, embedding := range embeddingList {
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
if len(embeddingDB.Embedding) == 0 {
log.Warnf("[Qdrant] Skipping empty embedding for chunk ID: %s", embedding.ChunkID)
continue
}
point := &qdrant.PointStruct{
Id: qdrant.NewID(uuid.New().String()),
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
Payload: createPayload(embeddingDB),
}
points = append(points, point)
log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID)
}
if len(points) == 0 {
log.Warn("[Qdrant] No valid points to save after filtering")
return nil
}
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
CollectionName: q.collectionName,
Points: points,
})
if err != nil {
log.Errorf("[Qdrant] Failed to execute batch operation: %v", err)
return fmt.Errorf("failed to batch save: %w", err)
}
log.Infof("[Qdrant] Successfully batch saved %d indices", len(points))
return nil
}
// DeleteByChunkIDList removes points from the collection based on chunk IDs
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
log := logger.GetLogger(ctx)
if len(chunkIDList) == 0 {
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
return nil
}
log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
CollectionName: q.collectionName,
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...),
},
}),
})
if err != nil {
log.Errorf("[Qdrant] Failed to delete by chunk IDs: %v", err)
return fmt.Errorf("failed to delete by chunk IDs: %w", err)
}
log.Infof("[Qdrant] Successfully deleted documents by chunk IDs")
return nil
}
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
) error {
log := logger.GetLogger(ctx)
if len(knowledgeIDList) == 0 {
log.Warn("[Qdrant] Empty knowledge ID list provided for deletion, skipping")
return nil
}
log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
CollectionName: q.collectionName,
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...),
},
}),
})
if err != nil {
log.Errorf("[Qdrant] Failed to delete by knowledge IDs: %v", err)
return fmt.Errorf("failed to delete by knowledge IDs: %w", err)
}
log.Infof("[Qdrant] Successfully deleted documents by knowledge IDs")
return nil
}
func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter {
must := make([]*qdrant.Condition, 0)
mustNot := make([]*qdrant.Condition, 0)
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
}
if len(params.ExcludeKnowledgeIDs) > 0 {
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
}
if len(params.ExcludeChunkIDs) > 0 {
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...))
}
return &qdrant.Filter{
Must: must,
MustNot: mustNot,
}
}
// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type
func (q *qdrantRepository) Retrieve(ctx context.Context,
params typesLocal.RetrieveParams,
) ([]*typesLocal.RetrieveResult, error) {
log := logger.GetLogger(ctx)
log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType)
switch params.RetrieverType {
case typesLocal.VectorRetrieverType:
return q.VectorRetrieve(ctx, params)
case typesLocal.KeywordsRetrieverType:
return q.KeywordsRetrieve(ctx, params)
}
err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType)
log.Errorf("[Qdrant] %v", err)
return nil, err
}
// VectorRetrieve performs vector similarity search
func (q *qdrantRepository) VectorRetrieve(ctx context.Context,
params typesLocal.RetrieveParams,
) ([]*typesLocal.RetrieveResult, error) {
log := logger.GetLogger(ctx)
log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
len(params.Embedding), params.TopK, params.Threshold)
filter := q.getBaseFilter(params)
limit := uint64(params.TopK)
scoreThreshold := float32(params.Threshold)
searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{
CollectionName: q.collectionName,
Query: qdrant.NewQuery(params.Embedding...),
Filter: filter,
Limit: &limit,
ScoreThreshold: &scoreThreshold,
WithPayload: qdrant.NewWithPayload(true),
})
if err != nil {
log.Errorf("[Qdrant] Vector search failed: %v", err)
return nil, err
}
var results []*typesLocal.IndexWithScore
for _, point := range searchResult {
payload := point.Payload
embedding := &QdrantVectorEmbeddingWithScore{
QdrantVectorEmbedding: QdrantVectorEmbedding{
Content: payload[fieldContent].GetStringValue(),
SourceID: payload[fieldSourceID].GetStringValue(),
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
ChunkID: payload[fieldChunkID].GetStringValue(),
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
},
Score: float64(point.Score),
}
pointID := point.Id.GetUuid()
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding))
}
if len(results) == 0 {
log.Warnf("[Qdrant] No vector matches found that meet threshold %.4f", params.Threshold)
} else {
log.Infof("[Qdrant] Vector retrieval found %d results", len(results))
log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score)
}
return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil
}
// KeywordsRetrieve performs keyword-based search in document content
func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context,
params typesLocal.RetrieveParams,
) ([]*typesLocal.RetrieveResult, error) {
log := logger.GetLogger(ctx)
log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK)
filter := q.getBaseFilter(params)
filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query))
limit := uint32(params.TopK)
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
CollectionName: q.collectionName,
Filter: filter,
Limit: &limit,
WithPayload: qdrant.NewWithPayload(true),
})
if err != nil {
log.Errorf("[Qdrant] Keywords search failed: %v", err)
return nil, err
}
var results []*typesLocal.IndexWithScore
for _, point := range scrollResult {
payload := point.Payload
embedding := &QdrantVectorEmbeddingWithScore{
QdrantVectorEmbedding: QdrantVectorEmbedding{
Content: payload[fieldContent].GetStringValue(),
SourceID: payload[fieldSourceID].GetStringValue(),
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
ChunkID: payload[fieldChunkID].GetStringValue(),
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
},
Score: 1.0,
}
pointID := point.Id.GetUuid()
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords))
}
if len(results) == 0 {
log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query)
} else {
log.Infof("[Qdrant] Keywords retrieval found %d results", len(results))
}
return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil
}
// CopyIndices copies index data from source knowledge base to target knowledge base
func (q *qdrantRepository) CopyIndices(ctx context.Context,
sourceKnowledgeBaseID string,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
) error {
log := logger.GetLogger(ctx)
log.Infof(
"[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d",
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
)
if len(sourceToTargetChunkIDMap) == 0 {
log.Warn("[Qdrant] Empty mapping, skipping copy")
return nil
}
batchSize := uint32(64)
var offset *qdrant.PointId = nil
totalCopied := 0
for {
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
CollectionName: q.collectionName,
Filter: &qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID),
},
},
Limit: &batchSize,
Offset: offset,
WithPayload: qdrant.NewWithPayload(true),
WithVectors: qdrant.NewWithVectors(true),
})
if err != nil {
log.Errorf("[Qdrant] Failed to query source points: %v", err)
return err
}
pointsCount := len(scrollResult)
if pointsCount == 0 {
break
}
log.Infof("[Qdrant] Found %d source points in batch", pointsCount)
targetPoints := make([]*qdrant.PointStruct, 0, pointsCount)
for _, sourcePoint := range scrollResult {
payload := sourcePoint.Payload
sourceChunkID := payload[fieldChunkID].GetStringValue()
sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue()
targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID]
if !ok {
log.Warnf("[Qdrant] Source chunk %s not found in target mapping, skipping", sourceChunkID)
continue
}
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceKnowledgeID]
if !ok {
log.Warnf("[Qdrant] Source knowledge %s not found in target mapping, skipping", sourceKnowledgeID)
continue
}
newPayload := qdrant.NewValueMap(map[string]any{
fieldContent: payload[fieldContent].GetStringValue(),
fieldSourceID: targetChunkID,
fieldSourceType: payload[fieldSourceType].GetIntegerValue(),
fieldChunkID: targetChunkID,
fieldKnowledgeID: targetKnowledgeID,
fieldKnowledgeBaseID: targetKnowledgeBaseID,
})
var vectors *qdrant.Vectors
if vectorOutput := sourcePoint.Vectors.GetVector(); vectorOutput != nil {
if denseVector := vectorOutput.GetDenseVector(); denseVector != nil {
vectors = qdrant.NewVectors(denseVector.Data...)
}
}
if vectors == nil {
log.Warnf("[Qdrant] No vectors found for source point with chunk %s, skipping", sourceChunkID)
continue
}
newPoint := &qdrant.PointStruct{
Id: qdrant.NewID(uuid.New().String()),
Vectors: vectors,
Payload: newPayload,
}
targetPoints = append(targetPoints, newPoint)
}
if len(targetPoints) > 0 {
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
CollectionName: q.collectionName,
Points: targetPoints,
})
if err != nil {
log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err)
return err
}
totalCopied += len(targetPoints)
log.Infof("[Qdrant] Successfully copied batch, batch size: %d, total copied: %d",
len(targetPoints), totalCopied)
}
if pointsCount > 0 {
offset = scrollResult[pointsCount-1].Id
}
if pointsCount < int(batchSize) {
break
}
}
log.Infof("[Qdrant] Index copy completed, total copied: %d", totalCopied)
return nil
}
func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value {
payload := map[string]any{
fieldContent: embedding.Content,
fieldSourceID: embedding.SourceID,
fieldSourceType: int64(embedding.SourceType),
fieldChunkID: embedding.ChunkID,
fieldKnowledgeID: embedding.KnowledgeID,
fieldKnowledgeBaseID: embedding.KnowledgeBaseID,
}
return qdrant.NewValueMap(payload)
}
func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult {
return []*typesLocal.RetrieveResult{
{
Results: results,
RetrieverEngineType: typesLocal.QdrantRetrieverEngineType,
RetrieverType: retrieverType,
Error: nil,
},
}
}
// Ref: https://github.com/qdrant/qdrant-sizing-calculator
func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding) int64 {
// Payload fields
payloadSizeBytes := int64(0)
payloadSizeBytes += int64(len(embedding.Content)) // content string
payloadSizeBytes += int64(len(embedding.SourceID)) // source_id string
payloadSizeBytes += int64(len(embedding.ChunkID)) // chunk_id string
payloadSizeBytes += int64(len(embedding.KnowledgeID)) // knowledge_id string
payloadSizeBytes += int64(len(embedding.KnowledgeBaseID)) // knowledge_base_id string
payloadSizeBytes += 8 // source_type int64
// Vector storage and index
var vectorSizeBytes int64 = 0
var hnswIndexBytes int64 = 0
if embedding.Embedding != nil {
dimensions := int64(len(embedding.Embedding))
vectorSizeBytes = dimensions * 4
// HNSW index: dimensions × (M × 2) × 4 bytes
// Default M=16, so: dimensions × 32 × 4 = dimensions × 128
const hnswM = 16
hnswIndexBytes = dimensions * (hnswM * 2) * 4
}
// ID tracker metadata: 24 bytes per vector
// (forward refs + backward refs + version tracking = 8 + 8 + 8)
const idTrackerBytes int64 = 24
totalSizeBytes := payloadSizeBytes + vectorSizeBytes + hnswIndexBytes + idTrackerBytes
return totalSizeBytes
}
// toQdrantVectorEmbedding converts IndexInfo to Qdrant payload format
func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *QdrantVectorEmbedding {
vector := &QdrantVectorEmbedding{
Content: embedding.Content,
SourceID: embedding.SourceID,
SourceType: int(embedding.SourceType),
ChunkID: embedding.ChunkID,
KnowledgeID: embedding.KnowledgeID,
KnowledgeBaseID: embedding.KnowledgeBaseID,
}
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok {
vector.Embedding = embeddingMap[embedding.SourceID]
}
}
return vector
}
// fromQdrantVectorEmbedding converts Qdrant point to IndexWithScore domain model
func fromQdrantVectorEmbedding(id string,
embedding *QdrantVectorEmbeddingWithScore,
matchType types.MatchType,
) *types.IndexWithScore {
return &types.IndexWithScore{
ID: id,
SourceID: embedding.SourceID,
SourceType: types.SourceType(embedding.SourceType),
ChunkID: embedding.ChunkID,
KnowledgeID: embedding.KnowledgeID,
KnowledgeBaseID: embedding.KnowledgeBaseID,
Content: embedding.Content,
Score: embedding.Score,
MatchType: matchType,
}
}
@@ -0,0 +1,23 @@
package qdrant
import "github.com/qdrant/go-client/qdrant"
type qdrantRepository struct {
client *qdrant.Client
collectionName string
}
type QdrantVectorEmbedding struct {
Content string `json:"content"`
SourceID string `json:"source_id"`
SourceType int `json:"source_type"`
ChunkID string `json:"chunk_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Embedding []float32 `json:"embedding"`
}
type QdrantVectorEmbeddingWithScore struct {
QdrantVectorEmbedding
Score float64
}
+6 -6
View File
@@ -11,9 +11,9 @@ import (
)
var (
ErrUserNotFound = errors.New("user not found")
ErrUserNotFound = errors.New("user not found")
ErrUserAlreadyExists = errors.New("user already exists")
ErrTokenNotFound = errors.New("token not found")
ErrTokenNotFound = errors.New("token not found")
)
// userRepository implements user repository interface
@@ -82,15 +82,15 @@ func (r *userRepository) DeleteUser(ctx context.Context, id string) error {
func (r *userRepository) ListUsers(ctx context.Context, offset, limit int) ([]*types.User, error) {
var users []*types.User
query := r.db.WithContext(ctx).Order("created_at DESC")
if limit > 0 {
query = query.Limit(limit)
}
if offset > 0 {
query = query.Offset(offset)
}
if err := query.Find(&users).Error; err != nil {
return nil, err
}
@@ -151,4 +151,4 @@ func (r *authTokenRepository) DeleteExpiredTokens(ctx context.Context) error {
// RevokeTokensByUserID revokes all tokens for a user
func (r *authTokenRepository) RevokeTokensByUserID(ctx context.Context, userID string) error {
return r.db.WithContext(ctx).Model(&types.AuthToken{}).Where("user_id = ?", userID).Update("is_revoked", true).Error
}
}
+50
View File
@@ -16,6 +16,7 @@ import (
"github.com/elastic/go-elasticsearch/v8"
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
"github.com/panjf2000/ants/v2"
"github.com/qdrant/go-client/qdrant"
"go.uber.org/dig"
"gorm.io/driver/postgres"
"gorm.io/gorm"
@@ -26,6 +27,7 @@ import (
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
"github.com/Tencent/WeKnora/internal/application/service"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/application/service/file"
@@ -330,6 +332,54 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
}
}
}
if slices.Contains(retrieveDriver, "qdrant") {
qdrantHost := os.Getenv("QDRANT_HOST")
if qdrantHost == "" {
qdrantHost = "localhost"
}
qdrantPort := 6334 // Default port
if portStr := os.Getenv("QDRANT_PORT"); portStr != "" {
if port, err := strconv.Atoi(portStr); err == nil {
qdrantPort = port
}
}
// API key for authentication (optional)
qdrantAPIKey := os.Getenv("QDRANT_API_KEY")
// TLS configuration (optional, defaults to false)
// Enable TLS unless explicitly set to "false" or "0" (case insensitive)
qdrantUseTLS := false
if useTLSStr := os.Getenv("QDRANT_USE_TLS"); useTLSStr != "" {
useTLSLower := strings.ToLower(strings.TrimSpace(useTLSStr))
qdrantUseTLS = useTLSLower != "false" && useTLSLower != "0"
}
log.Infof("Connecting to Qdrant at %s:%d (TLS: %v)", qdrantHost, qdrantPort, qdrantUseTLS)
client, err := qdrant.NewClient(&qdrant.Config{
Host: qdrantHost,
Port: qdrantPort,
APIKey: qdrantAPIKey,
UseTLS: qdrantUseTLS,
})
if err != nil {
log.Errorf("Create qdrant client failed: %v", err)
} else {
qdrantRepository := qdrantRepo.NewQdrantRetrieveEngineRepository(client)
if err := registry.Register(
retriever.NewKVHybridRetrieveEngine(
qdrantRepository, types.QdrantRetrieverEngineType,
),
); err != nil {
log.Errorf("Register qdrant retrieve engine failed: %v", err)
} else {
log.Infof("Register qdrant retrieve engine success")
}
}
}
return registry, nil
}
+1
View File
@@ -9,6 +9,7 @@ const (
ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch"
InfinityRetrieverEngineType RetrieverEngineType = "infinity"
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
)
// RetrieverType represents the type of retriever