feat: Qdrant Vector Search support
Signed-off-by: Anush008 <anushshetty90@gmail.com>
This commit is contained in:
+17
-1
@@ -14,7 +14,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
# 主数据库类型(postgres/mysql)
|
||||
DB_DRIVER=postgres
|
||||
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8)
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
|
||||
RETRIEVE_DRIVER=postgres
|
||||
|
||||
# 文件存储类型(local/minio/cos)
|
||||
@@ -81,6 +81,22 @@ CONCURRENCY_POOL_SIZE=5
|
||||
# ElasticSearch索引名称,用于存储向量数据
|
||||
# ELASTICSEARCH_INDEX=WeKnora
|
||||
|
||||
# 如果使用Qdrant作为向量存储,需要配置以下参数
|
||||
# Qdrant服务主机地址
|
||||
# QDRANT_HOST=localhost
|
||||
|
||||
# Qdrant服务端口
|
||||
# QDRANT_PORT=6334
|
||||
|
||||
# Qdrant集合名称,用于存储向量数据
|
||||
# QDRANT_COLLECTION=weknora_embeddings
|
||||
|
||||
# Qdrant API密钥,如果需要身份验证(可选)
|
||||
# QDRANT_API_KEY=your_qdrant_api_key
|
||||
|
||||
# 是否启用TLS加密连接(可选,默认为false)
|
||||
# QDRANT_USE_TLS=false
|
||||
|
||||
# 如果使用MinIO作为文件存储,需要配置以下参数
|
||||
# MinIO访问密钥
|
||||
# MINIO_ACCESS_KEY_ID=your_minio_access_key
|
||||
|
||||
@@ -19,6 +19,7 @@ require (
|
||||
github.com/panjf2000/ants/v2 v2.11.2
|
||||
github.com/parquet-go/parquet-go v0.25.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/qdrant/go-client v1.16.1
|
||||
github.com/redis/go-redis/v9 v9.14.0
|
||||
github.com/sashabaranov/go-openai v1.40.5
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
@@ -26,17 +27,17 @@ require (
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
|
||||
github.com/yanyiwu/gojieba v1.4.5
|
||||
go.opentelemetry.io/otel v1.37.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0
|
||||
go.opentelemetry.io/otel/sdk v1.37.0
|
||||
go.opentelemetry.io/otel/trace v1.37.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
go.uber.org/dig v1.18.1
|
||||
golang.org/x/crypto v0.42.0
|
||||
golang.org/x/sync v0.17.0
|
||||
google.golang.org/grpc v1.73.0
|
||||
google.golang.org/protobuf v1.36.9
|
||||
golang.org/x/crypto v0.44.0
|
||||
golang.org/x/sync v0.18.0
|
||||
google.golang.org/grpc v1.76.0
|
||||
google.golang.org/protobuf v1.36.10
|
||||
gorm.io/driver/postgres v1.5.11
|
||||
gorm.io/gorm v1.25.12
|
||||
)
|
||||
@@ -72,7 +73,7 @@ require (
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
@@ -98,16 +99,16 @@ require (
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
golang.org/x/time v0.13.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -106,8 +106,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
|
||||
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
@@ -159,6 +159,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/qdrant/go-client v1.16.1 h1:Jr47kz0k8I+U2sUm2UUO2eq2kL0fTcgjLPIz6a0RKuQ=
|
||||
github.com/qdrant/go-client v1.16.1/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw=
|
||||
github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE=
|
||||
github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
@@ -166,8 +168,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
|
||||
@@ -229,24 +231,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI=
|
||||
github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgTNQVet8iTDW7oIk03tXHq+wkwIDnE=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
|
||||
go.uber.org/dig v1.18.1 h1:rLww6NuajVjeQn+49u5NcezUJEGwd5uXmyoCKW2g5Es=
|
||||
@@ -257,31 +259,33 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
|
||||
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
|
||||
golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
|
||||
golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
|
||||
golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
|
||||
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
|
||||
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
|
||||
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
|
||||
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:oDOGiMSXHL4sDTJvFvIB9nRQCGdLP1o/iVaqQK8zB+M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
|
||||
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@@ -0,0 +1,572 @@
|
||||
package qdrant
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
typesLocal "github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
const (
|
||||
envQdrantCollection = "QDRANT_COLLECTION"
|
||||
defaultCollectionName = "weknora_embeddings"
|
||||
fieldContent = "content"
|
||||
fieldSourceID = "source_id"
|
||||
fieldSourceType = "source_type"
|
||||
fieldChunkID = "chunk_id"
|
||||
fieldKnowledgeID = "knowledge_id"
|
||||
fieldKnowledgeBaseID = "knowledge_base_id"
|
||||
fieldEmbedding = "embedding"
|
||||
)
|
||||
|
||||
// NewQdrantRetrieveEngineRepository creates and initializes a new Qdrant repository
|
||||
func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.RetrieveEngineRepository {
|
||||
log := logger.GetLogger(context.Background())
|
||||
log.Info("[Qdrant] Initializing Qdrant retriever engine repository")
|
||||
|
||||
collectionName := os.Getenv(envQdrantCollection)
|
||||
if collectionName == "" {
|
||||
log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name")
|
||||
collectionName = defaultCollectionName
|
||||
}
|
||||
|
||||
res := &qdrantRepository{
|
||||
client: client,
|
||||
collectionName: collectionName,
|
||||
}
|
||||
|
||||
log.Info("[Qdrant] Successfully initialized repository")
|
||||
return res
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType {
|
||||
return typesLocal.QdrantRetrieverEngineType
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) Support() []typesLocal.RetrieverType {
|
||||
return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType}
|
||||
}
|
||||
|
||||
// EstimateStorageSize calculates the estimated storage size for a list of indices
|
||||
func (q *qdrantRepository) EstimateStorageSize(ctx context.Context,
|
||||
indexInfoList []*typesLocal.IndexInfo, params map[string]any,
|
||||
) int64 {
|
||||
var totalStorageSize int64
|
||||
for _, embedding := range indexInfoList {
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, params)
|
||||
totalStorageSize += q.calculateStorageSize(embeddingDB)
|
||||
}
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"[Qdrant] Storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize,
|
||||
)
|
||||
return totalStorageSize
|
||||
}
|
||||
|
||||
// Save stores a single point in Qdrant
|
||||
func (q *qdrantRepository) Save(ctx context.Context,
|
||||
embedding *typesLocal.IndexInfo,
|
||||
additionalParams map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Debugf("[Qdrant] Saving index for chunk ID: %s", embedding.ChunkID)
|
||||
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
|
||||
if len(embeddingDB.Embedding) == 0 {
|
||||
err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID)
|
||||
log.Errorf("[Qdrant] %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
pointID := uuid.New().String()
|
||||
point := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(pointID),
|
||||
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
|
||||
Payload: createPayload(embeddingDB),
|
||||
}
|
||||
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: []*qdrant.PointStruct{point},
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to save index: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchSave stores multiple points in Qdrant using batch upsert
|
||||
func (q *qdrantRepository) BatchSave(ctx context.Context,
|
||||
embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(embeddingList) == 0 {
|
||||
log.Warn("[Qdrant] Empty list provided to BatchSave, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList))
|
||||
|
||||
points := make([]*qdrant.PointStruct, 0, len(embeddingList))
|
||||
for _, embedding := range embeddingList {
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
|
||||
if len(embeddingDB.Embedding) == 0 {
|
||||
log.Warnf("[Qdrant] Skipping empty embedding for chunk ID: %s", embedding.ChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
point := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
|
||||
Payload: createPayload(embeddingDB),
|
||||
}
|
||||
points = append(points, point)
|
||||
log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID)
|
||||
}
|
||||
|
||||
if len(points) == 0 {
|
||||
log.Warn("[Qdrant] No valid points to save after filtering")
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: points,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to execute batch operation: %v", err)
|
||||
return fmt.Errorf("failed to batch save: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully batch saved %d indices", len(points))
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList removes points from the collection based on chunk IDs
|
||||
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkIDList) == 0 {
|
||||
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
|
||||
|
||||
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to delete by chunk IDs: %v", err)
|
||||
return fmt.Errorf("failed to delete by chunk IDs: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully deleted documents by chunk IDs")
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
|
||||
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(knowledgeIDList) == 0 {
|
||||
log.Warn("[Qdrant] Empty knowledge ID list provided for deletion, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
|
||||
|
||||
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to delete by knowledge IDs: %v", err)
|
||||
return fmt.Errorf("failed to delete by knowledge IDs: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully deleted documents by knowledge IDs")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter {
|
||||
must := make([]*qdrant.Condition, 0)
|
||||
mustNot := make([]*qdrant.Condition, 0)
|
||||
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...))
|
||||
}
|
||||
|
||||
return &qdrant.Filter{
|
||||
Must: must,
|
||||
MustNot: mustNot,
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type
|
||||
func (q *qdrantRepository) Retrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType)
|
||||
|
||||
switch params.RetrieverType {
|
||||
case typesLocal.VectorRetrieverType:
|
||||
return q.VectorRetrieve(ctx, params)
|
||||
case typesLocal.KeywordsRetrieverType:
|
||||
return q.KeywordsRetrieve(ctx, params)
|
||||
}
|
||||
|
||||
err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType)
|
||||
log.Errorf("[Qdrant] %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VectorRetrieve performs vector similarity search
|
||||
func (q *qdrantRepository) VectorRetrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
|
||||
len(params.Embedding), params.TopK, params.Threshold)
|
||||
|
||||
filter := q.getBaseFilter(params)
|
||||
|
||||
limit := uint64(params.TopK)
|
||||
scoreThreshold := float32(params.Threshold)
|
||||
|
||||
searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Query: qdrant.NewQuery(params.Embedding...),
|
||||
Filter: filter,
|
||||
Limit: &limit,
|
||||
ScoreThreshold: &scoreThreshold,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Vector search failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*typesLocal.IndexWithScore
|
||||
for _, point := range searchResult {
|
||||
payload := point.Payload
|
||||
embedding := &QdrantVectorEmbeddingWithScore{
|
||||
QdrantVectorEmbedding: QdrantVectorEmbedding{
|
||||
Content: payload[fieldContent].GetStringValue(),
|
||||
SourceID: payload[fieldSourceID].GetStringValue(),
|
||||
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
},
|
||||
Score: float64(point.Score),
|
||||
}
|
||||
|
||||
pointID := point.Id.GetUuid()
|
||||
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding))
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
log.Warnf("[Qdrant] No vector matches found that meet threshold %.4f", params.Threshold)
|
||||
} else {
|
||||
log.Infof("[Qdrant] Vector retrieval found %d results", len(results))
|
||||
log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score)
|
||||
}
|
||||
|
||||
return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil
|
||||
}
|
||||
|
||||
// KeywordsRetrieve performs keyword-based search in document content
|
||||
func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK)
|
||||
|
||||
filter := q.getBaseFilter(params)
|
||||
|
||||
filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query))
|
||||
|
||||
limit := uint32(params.TopK)
|
||||
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Filter: filter,
|
||||
Limit: &limit,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Keywords search failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*typesLocal.IndexWithScore
|
||||
for _, point := range scrollResult {
|
||||
payload := point.Payload
|
||||
embedding := &QdrantVectorEmbeddingWithScore{
|
||||
QdrantVectorEmbedding: QdrantVectorEmbedding{
|
||||
Content: payload[fieldContent].GetStringValue(),
|
||||
SourceID: payload[fieldSourceID].GetStringValue(),
|
||||
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
},
|
||||
Score: 1.0,
|
||||
}
|
||||
|
||||
pointID := point.Id.GetUuid()
|
||||
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords))
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query)
|
||||
} else {
|
||||
log.Infof("[Qdrant] Keywords retrieval found %d results", len(results))
|
||||
}
|
||||
|
||||
return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
// CopyIndices copies index data from source knowledge base to target knowledge base
|
||||
func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
sourceKnowledgeBaseID string,
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof(
|
||||
"[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d",
|
||||
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
|
||||
)
|
||||
|
||||
if len(sourceToTargetChunkIDMap) == 0 {
|
||||
log.Warn("[Qdrant] Empty mapping, skipping copy")
|
||||
return nil
|
||||
}
|
||||
|
||||
batchSize := uint32(64)
|
||||
var offset *qdrant.PointId = nil
|
||||
totalCopied := 0
|
||||
|
||||
for {
|
||||
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Filter: &qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID),
|
||||
},
|
||||
},
|
||||
Limit: &batchSize,
|
||||
Offset: offset,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
WithVectors: qdrant.NewWithVectors(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to query source points: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
pointsCount := len(scrollResult)
|
||||
if pointsCount == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Found %d source points in batch", pointsCount)
|
||||
|
||||
targetPoints := make([]*qdrant.PointStruct, 0, pointsCount)
|
||||
for _, sourcePoint := range scrollResult {
|
||||
payload := sourcePoint.Payload
|
||||
|
||||
sourceChunkID := payload[fieldChunkID].GetStringValue()
|
||||
sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue()
|
||||
|
||||
targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID]
|
||||
if !ok {
|
||||
log.Warnf("[Qdrant] Source chunk %s not found in target mapping, skipping", sourceChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceKnowledgeID]
|
||||
if !ok {
|
||||
log.Warnf("[Qdrant] Source knowledge %s not found in target mapping, skipping", sourceKnowledgeID)
|
||||
continue
|
||||
}
|
||||
|
||||
newPayload := qdrant.NewValueMap(map[string]any{
|
||||
fieldContent: payload[fieldContent].GetStringValue(),
|
||||
fieldSourceID: targetChunkID,
|
||||
fieldSourceType: payload[fieldSourceType].GetIntegerValue(),
|
||||
fieldChunkID: targetChunkID,
|
||||
fieldKnowledgeID: targetKnowledgeID,
|
||||
fieldKnowledgeBaseID: targetKnowledgeBaseID,
|
||||
})
|
||||
|
||||
var vectors *qdrant.Vectors
|
||||
if vectorOutput := sourcePoint.Vectors.GetVector(); vectorOutput != nil {
|
||||
if denseVector := vectorOutput.GetDenseVector(); denseVector != nil {
|
||||
vectors = qdrant.NewVectors(denseVector.Data...)
|
||||
}
|
||||
}
|
||||
|
||||
if vectors == nil {
|
||||
log.Warnf("[Qdrant] No vectors found for source point with chunk %s, skipping", sourceChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
newPoint := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: vectors,
|
||||
Payload: newPayload,
|
||||
}
|
||||
|
||||
targetPoints = append(targetPoints, newPoint)
|
||||
}
|
||||
|
||||
if len(targetPoints) > 0 {
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: targetPoints,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
totalCopied += len(targetPoints)
|
||||
log.Infof("[Qdrant] Successfully copied batch, batch size: %d, total copied: %d",
|
||||
len(targetPoints), totalCopied)
|
||||
}
|
||||
|
||||
if pointsCount > 0 {
|
||||
offset = scrollResult[pointsCount-1].Id
|
||||
}
|
||||
|
||||
if pointsCount < int(batchSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Index copy completed, total copied: %d", totalCopied)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value {
|
||||
payload := map[string]any{
|
||||
fieldContent: embedding.Content,
|
||||
fieldSourceID: embedding.SourceID,
|
||||
fieldSourceType: int64(embedding.SourceType),
|
||||
fieldChunkID: embedding.ChunkID,
|
||||
fieldKnowledgeID: embedding.KnowledgeID,
|
||||
fieldKnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
}
|
||||
return qdrant.NewValueMap(payload)
|
||||
}
|
||||
|
||||
func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult {
|
||||
return []*typesLocal.RetrieveResult{
|
||||
{
|
||||
Results: results,
|
||||
RetrieverEngineType: typesLocal.QdrantRetrieverEngineType,
|
||||
RetrieverType: retrieverType,
|
||||
Error: nil,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Ref: https://github.com/qdrant/qdrant-sizing-calculator
|
||||
func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding) int64 {
|
||||
// Payload fields
|
||||
payloadSizeBytes := int64(0)
|
||||
payloadSizeBytes += int64(len(embedding.Content)) // content string
|
||||
payloadSizeBytes += int64(len(embedding.SourceID)) // source_id string
|
||||
payloadSizeBytes += int64(len(embedding.ChunkID)) // chunk_id string
|
||||
payloadSizeBytes += int64(len(embedding.KnowledgeID)) // knowledge_id string
|
||||
payloadSizeBytes += int64(len(embedding.KnowledgeBaseID)) // knowledge_base_id string
|
||||
payloadSizeBytes += 8 // source_type int64
|
||||
|
||||
// Vector storage and index
|
||||
var vectorSizeBytes int64 = 0
|
||||
var hnswIndexBytes int64 = 0
|
||||
if embedding.Embedding != nil {
|
||||
dimensions := int64(len(embedding.Embedding))
|
||||
vectorSizeBytes = dimensions * 4
|
||||
|
||||
// HNSW index: dimensions × (M × 2) × 4 bytes
|
||||
// Default M=16, so: dimensions × 32 × 4 = dimensions × 128
|
||||
const hnswM = 16
|
||||
hnswIndexBytes = dimensions * (hnswM * 2) * 4
|
||||
}
|
||||
|
||||
// ID tracker metadata: 24 bytes per vector
|
||||
// (forward refs + backward refs + version tracking = 8 + 8 + 8)
|
||||
const idTrackerBytes int64 = 24
|
||||
|
||||
totalSizeBytes := payloadSizeBytes + vectorSizeBytes + hnswIndexBytes + idTrackerBytes
|
||||
return totalSizeBytes
|
||||
}
|
||||
|
||||
// toQdrantVectorEmbedding converts IndexInfo to Qdrant payload format
|
||||
func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *QdrantVectorEmbedding {
|
||||
vector := &QdrantVectorEmbedding{
|
||||
Content: embedding.Content,
|
||||
SourceID: embedding.SourceID,
|
||||
SourceType: int(embedding.SourceType),
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
}
|
||||
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
|
||||
if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok {
|
||||
vector.Embedding = embeddingMap[embedding.SourceID]
|
||||
}
|
||||
}
|
||||
return vector
|
||||
}
|
||||
|
||||
// fromQdrantVectorEmbedding converts Qdrant point to IndexWithScore domain model
|
||||
func fromQdrantVectorEmbedding(id string,
|
||||
embedding *QdrantVectorEmbeddingWithScore,
|
||||
matchType types.MatchType,
|
||||
) *types.IndexWithScore {
|
||||
return &types.IndexWithScore{
|
||||
ID: id,
|
||||
SourceID: embedding.SourceID,
|
||||
SourceType: types.SourceType(embedding.SourceType),
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
Content: embedding.Content,
|
||||
Score: embedding.Score,
|
||||
MatchType: matchType,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package qdrant
|
||||
|
||||
import "github.com/qdrant/go-client/qdrant"
|
||||
|
||||
type qdrantRepository struct {
|
||||
client *qdrant.Client
|
||||
collectionName string
|
||||
}
|
||||
|
||||
type QdrantVectorEmbedding struct {
|
||||
Content string `json:"content"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceType int `json:"source_type"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
}
|
||||
|
||||
type QdrantVectorEmbeddingWithScore struct {
|
||||
QdrantVectorEmbedding
|
||||
Score float64
|
||||
}
|
||||
@@ -11,9 +11,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUserNotFound = errors.New("user not found")
|
||||
ErrUserNotFound = errors.New("user not found")
|
||||
ErrUserAlreadyExists = errors.New("user already exists")
|
||||
ErrTokenNotFound = errors.New("token not found")
|
||||
ErrTokenNotFound = errors.New("token not found")
|
||||
)
|
||||
|
||||
// userRepository implements user repository interface
|
||||
@@ -82,15 +82,15 @@ func (r *userRepository) DeleteUser(ctx context.Context, id string) error {
|
||||
func (r *userRepository) ListUsers(ctx context.Context, offset, limit int) ([]*types.User, error) {
|
||||
var users []*types.User
|
||||
query := r.db.WithContext(ctx).Order("created_at DESC")
|
||||
|
||||
|
||||
if limit > 0 {
|
||||
query = query.Limit(limit)
|
||||
}
|
||||
|
||||
|
||||
if offset > 0 {
|
||||
query = query.Offset(offset)
|
||||
}
|
||||
|
||||
|
||||
if err := query.Find(&users).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -151,4 +151,4 @@ func (r *authTokenRepository) DeleteExpiredTokens(ctx context.Context) error {
|
||||
// RevokeTokensByUserID revokes all tokens for a user
|
||||
func (r *authTokenRepository) RevokeTokensByUserID(ctx context.Context, userID string) error {
|
||||
return r.db.WithContext(ctx).Model(&types.AuthToken{}).Where("user_id = ?", userID).Update("is_revoked", true).Error
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"go.uber.org/dig"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
@@ -26,6 +27,7 @@ import (
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
@@ -330,6 +332,54 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if slices.Contains(retrieveDriver, "qdrant") {
|
||||
qdrantHost := os.Getenv("QDRANT_HOST")
|
||||
if qdrantHost == "" {
|
||||
qdrantHost = "localhost"
|
||||
}
|
||||
|
||||
qdrantPort := 6334 // Default port
|
||||
if portStr := os.Getenv("QDRANT_PORT"); portStr != "" {
|
||||
if port, err := strconv.Atoi(portStr); err == nil {
|
||||
qdrantPort = port
|
||||
}
|
||||
}
|
||||
|
||||
// API key for authentication (optional)
|
||||
qdrantAPIKey := os.Getenv("QDRANT_API_KEY")
|
||||
|
||||
// TLS configuration (optional, defaults to false)
|
||||
// Enable TLS unless explicitly set to "false" or "0" (case insensitive)
|
||||
qdrantUseTLS := false
|
||||
if useTLSStr := os.Getenv("QDRANT_USE_TLS"); useTLSStr != "" {
|
||||
useTLSLower := strings.ToLower(strings.TrimSpace(useTLSStr))
|
||||
qdrantUseTLS = useTLSLower != "false" && useTLSLower != "0"
|
||||
}
|
||||
|
||||
log.Infof("Connecting to Qdrant at %s:%d (TLS: %v)", qdrantHost, qdrantPort, qdrantUseTLS)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: qdrantHost,
|
||||
Port: qdrantPort,
|
||||
APIKey: qdrantAPIKey,
|
||||
UseTLS: qdrantUseTLS,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("Create qdrant client failed: %v", err)
|
||||
} else {
|
||||
qdrantRepository := qdrantRepo.NewQdrantRetrieveEngineRepository(client)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(
|
||||
qdrantRepository, types.QdrantRetrieverEngineType,
|
||||
),
|
||||
); err != nil {
|
||||
log.Errorf("Register qdrant retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register qdrant retrieve engine success")
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ const (
|
||||
ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch"
|
||||
InfinityRetrieverEngineType RetrieverEngineType = "infinity"
|
||||
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
|
||||
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
|
||||
)
|
||||
|
||||
// RetrieverType represents the type of retriever
|
||||
|
||||
Reference in New Issue
Block a user