diff --git a/.env.example b/.env.example index e6c239ed..b6f4c59c 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # 主数据库类型(postgres/mysql) DB_DRIVER=postgres -# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8) +# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant) RETRIEVE_DRIVER=postgres # 文件存储类型(local/minio/cos) @@ -81,6 +81,22 @@ CONCURRENCY_POOL_SIZE=5 # ElasticSearch索引名称,用于存储向量数据 # ELASTICSEARCH_INDEX=WeKnora +# 如果使用Qdrant作为向量存储,需要配置以下参数 +# Qdrant服务主机地址 +# QDRANT_HOST=localhost + +# Qdrant服务端口 +# QDRANT_PORT=6334 + +# Qdrant集合名称,用于存储向量数据 +# QDRANT_COLLECTION=weknora_embeddings + +# Qdrant API密钥,如果需要身份验证(可选) +# QDRANT_API_KEY=your_qdrant_api_key + +# 是否启用TLS加密连接(可选,默认为false) +# QDRANT_USE_TLS=false + # 如果使用MinIO作为文件存储,需要配置以下参数 # MinIO访问密钥 # MINIO_ACCESS_KEY_ID=your_minio_access_key diff --git a/go.mod b/go.mod index 1675d542..a3ed88f4 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/panjf2000/ants/v2 v2.11.2 github.com/parquet-go/parquet-go v0.25.0 github.com/pgvector/pgvector-go v0.3.0 + github.com/qdrant/go-client v1.16.1 github.com/redis/go-redis/v9 v9.14.0 github.com/sashabaranov/go-openai v1.40.5 github.com/sirupsen/logrus v1.9.3 @@ -26,17 +27,17 @@ require ( github.com/stretchr/testify v1.11.1 github.com/tencentyun/cos-go-sdk-v5 v0.7.65 github.com/yanyiwu/gojieba v1.4.5 - go.opentelemetry.io/otel v1.37.0 + go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 go.opentelemetry.io/otel/sdk v1.37.0 - go.opentelemetry.io/otel/trace v1.37.0 + go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/dig v1.18.1 - golang.org/x/crypto v0.42.0 - golang.org/x/sync v0.17.0 - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.9 + golang.org/x/crypto v0.44.0 + golang.org/x/sync v0.18.0 + google.golang.org/grpc v1.76.0 + google.golang.org/protobuf v1.36.10 gorm.io/driver/postgres v1.5.11 gorm.io/gorm v1.25.12 ) @@ -72,7 +73,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.1 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -98,16 +99,16 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.15.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.29.0 // indirect - golang.org/x/time v0.13.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e8a3ae10..1e806fa9 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= +github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= @@ -159,6 +159,8 @@ github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qdrant/go-client v1.16.1 h1:Jr47kz0k8I+U2sUm2UUO2eq2kL0fTcgjLPIz6a0RKuQ= +github.com/qdrant/go-client v1.16.1/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw= github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE= github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -166,8 +168,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= @@ -229,24 +231,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI= github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgTNQVet8iTDW7oIk03tXHq+wkwIDnE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/dig v1.18.1 h1:rLww6NuajVjeQn+49u5NcezUJEGwd5uXmyoCKW2g5Es= @@ -257,31 +259,33 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= -golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= -google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc= +google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:oDOGiMSXHL4sDTJvFvIB9nRQCGdLP1o/iVaqQK8zB+M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go new file mode 100644 index 00000000..284cd317 --- /dev/null +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -0,0 +1,572 @@ +package qdrant + +import ( + "context" + "fmt" + "maps" + "os" + "slices" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + typesLocal "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/qdrant/go-client/qdrant" +) + +const ( + envQdrantCollection = "QDRANT_COLLECTION" + defaultCollectionName = "weknora_embeddings" + fieldContent = "content" + fieldSourceID = "source_id" + fieldSourceType = "source_type" + fieldChunkID = "chunk_id" + fieldKnowledgeID = "knowledge_id" + fieldKnowledgeBaseID = "knowledge_base_id" + fieldEmbedding = "embedding" +) + +// NewQdrantRetrieveEngineRepository creates and initializes a new Qdrant repository +func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.RetrieveEngineRepository { + log := logger.GetLogger(context.Background()) + log.Info("[Qdrant] Initializing Qdrant retriever engine repository") + + collectionName := os.Getenv(envQdrantCollection) + if collectionName == "" { + log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name") + collectionName = defaultCollectionName + } + + res := &qdrantRepository{ + client: client, + collectionName: collectionName, + } + + log.Info("[Qdrant] Successfully initialized repository") + return res +} + +func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType { + return typesLocal.QdrantRetrieverEngineType +} + +func (q *qdrantRepository) Support() []typesLocal.RetrieverType { + return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType} +} + +// EstimateStorageSize calculates the estimated storage size for a list of indices +func (q *qdrantRepository) EstimateStorageSize(ctx context.Context, + indexInfoList []*typesLocal.IndexInfo, params map[string]any, +) int64 { + var totalStorageSize int64 + for _, embedding := range indexInfoList { + embeddingDB := toQdrantVectorEmbedding(embedding, params) + totalStorageSize += q.calculateStorageSize(embeddingDB) + } + logger.GetLogger(ctx).Infof( + "[Qdrant] Storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize, + ) + return totalStorageSize +} + +// Save stores a single point in Qdrant +func (q *qdrantRepository) Save(ctx context.Context, + embedding *typesLocal.IndexInfo, + additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + log.Debugf("[Qdrant] Saving index for chunk ID: %s", embedding.ChunkID) + + embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams) + if len(embeddingDB.Embedding) == 0 { + err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID) + log.Errorf("[Qdrant] %v", err) + return err + } + + pointID := uuid.New().String() + point := &qdrant.PointStruct{ + Id: qdrant.NewID(pointID), + Vectors: qdrant.NewVectors(embeddingDB.Embedding...), + Payload: createPayload(embeddingDB), + } + + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: q.collectionName, + Points: []*qdrant.PointStruct{point}, + }) + if err != nil { + log.Errorf("[Qdrant] Failed to save index: %v", err) + return err + } + + log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID) + return nil +} + +// BatchSave stores multiple points in Qdrant using batch upsert +func (q *qdrantRepository) BatchSave(ctx context.Context, + embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + if len(embeddingList) == 0 { + log.Warn("[Qdrant] Empty list provided to BatchSave, skipping") + return nil + } + + log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList)) + + points := make([]*qdrant.PointStruct, 0, len(embeddingList)) + for _, embedding := range embeddingList { + embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams) + if len(embeddingDB.Embedding) == 0 { + log.Warnf("[Qdrant] Skipping empty embedding for chunk ID: %s", embedding.ChunkID) + continue + } + + point := &qdrant.PointStruct{ + Id: qdrant.NewID(uuid.New().String()), + Vectors: qdrant.NewVectors(embeddingDB.Embedding...), + Payload: createPayload(embeddingDB), + } + points = append(points, point) + log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID) + } + + if len(points) == 0 { + log.Warn("[Qdrant] No valid points to save after filtering") + return nil + } + + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: q.collectionName, + Points: points, + }) + if err != nil { + log.Errorf("[Qdrant] Failed to execute batch operation: %v", err) + return fmt.Errorf("failed to batch save: %w", err) + } + + log.Infof("[Qdrant] Successfully batch saved %d indices", len(points)) + return nil +} + +// DeleteByChunkIDList removes points from the collection based on chunk IDs +func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { + log := logger.GetLogger(ctx) + if len(chunkIDList) == 0 { + log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping") + return nil + } + + log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) + + _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ + CollectionName: q.collectionName, + Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...), + }, + }), + }) + if err != nil { + log.Errorf("[Qdrant] Failed to delete by chunk IDs: %v", err) + return fmt.Errorf("failed to delete by chunk IDs: %w", err) + } + + log.Infof("[Qdrant] Successfully deleted documents by chunk IDs") + return nil +} + +// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs +func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context, + knowledgeIDList []string, dimension int, +) error { + log := logger.GetLogger(ctx) + if len(knowledgeIDList) == 0 { + log.Warn("[Qdrant] Empty knowledge ID list provided for deletion, skipping") + return nil + } + + log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) + + _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ + CollectionName: q.collectionName, + Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...), + }, + }), + }) + if err != nil { + log.Errorf("[Qdrant] Failed to delete by knowledge IDs: %v", err) + return fmt.Errorf("failed to delete by knowledge IDs: %w", err) + } + + log.Infof("[Qdrant] Successfully deleted documents by knowledge IDs") + return nil +} + +func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter { + must := make([]*qdrant.Condition, 0) + mustNot := make([]*qdrant.Condition, 0) + + if len(params.KnowledgeBaseIDs) > 0 { + must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...)) + } + + if len(params.ExcludeKnowledgeIDs) > 0 { + mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...)) + } + + if len(params.ExcludeChunkIDs) > 0 { + mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...)) + } + + return &qdrant.Filter{ + Must: must, + MustNot: mustNot, + } +} + +// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type +func (q *qdrantRepository) Retrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType) + + switch params.RetrieverType { + case typesLocal.VectorRetrieverType: + return q.VectorRetrieve(ctx, params) + case typesLocal.KeywordsRetrieverType: + return q.KeywordsRetrieve(ctx, params) + } + + err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType) + log.Errorf("[Qdrant] %v", err) + return nil, err +} + +// VectorRetrieve performs vector similarity search +func (q *qdrantRepository) VectorRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", + len(params.Embedding), params.TopK, params.Threshold) + + filter := q.getBaseFilter(params) + + limit := uint64(params.TopK) + scoreThreshold := float32(params.Threshold) + + searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{ + CollectionName: q.collectionName, + Query: qdrant.NewQuery(params.Embedding...), + Filter: filter, + Limit: &limit, + ScoreThreshold: &scoreThreshold, + WithPayload: qdrant.NewWithPayload(true), + }) + if err != nil { + log.Errorf("[Qdrant] Vector search failed: %v", err) + return nil, err + } + + var results []*typesLocal.IndexWithScore + for _, point := range searchResult { + payload := point.Payload + embedding := &QdrantVectorEmbeddingWithScore{ + QdrantVectorEmbedding: QdrantVectorEmbedding{ + Content: payload[fieldContent].GetStringValue(), + SourceID: payload[fieldSourceID].GetStringValue(), + SourceType: int(payload[fieldSourceType].GetIntegerValue()), + ChunkID: payload[fieldChunkID].GetStringValue(), + KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), + KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + }, + Score: float64(point.Score), + } + + pointID := point.Id.GetUuid() + results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding)) + } + + if len(results) == 0 { + log.Warnf("[Qdrant] No vector matches found that meet threshold %.4f", params.Threshold) + } else { + log.Infof("[Qdrant] Vector retrieval found %d results", len(results)) + log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score) + } + + return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil +} + +// KeywordsRetrieve performs keyword-based search in document content +func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK) + + filter := q.getBaseFilter(params) + + filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query)) + + limit := uint32(params.TopK) + scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ + CollectionName: q.collectionName, + Filter: filter, + Limit: &limit, + WithPayload: qdrant.NewWithPayload(true), + }) + if err != nil { + log.Errorf("[Qdrant] Keywords search failed: %v", err) + return nil, err + } + + var results []*typesLocal.IndexWithScore + for _, point := range scrollResult { + payload := point.Payload + embedding := &QdrantVectorEmbeddingWithScore{ + QdrantVectorEmbedding: QdrantVectorEmbedding{ + Content: payload[fieldContent].GetStringValue(), + SourceID: payload[fieldSourceID].GetStringValue(), + SourceType: int(payload[fieldSourceType].GetIntegerValue()), + ChunkID: payload[fieldChunkID].GetStringValue(), + KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), + KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + }, + Score: 1.0, + } + + pointID := point.Id.GetUuid() + results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords)) + } + + if len(results) == 0 { + log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query) + } else { + log.Infof("[Qdrant] Keywords retrieval found %d results", len(results)) + } + + return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil +} + +// CopyIndices copies index data from source knowledge base to target knowledge base +func (q *qdrantRepository) CopyIndices(ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, +) error { + log := logger.GetLogger(ctx) + log.Infof( + "[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + ) + + if len(sourceToTargetChunkIDMap) == 0 { + log.Warn("[Qdrant] Empty mapping, skipping copy") + return nil + } + + batchSize := uint32(64) + var offset *qdrant.PointId = nil + totalCopied := 0 + + for { + scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ + CollectionName: q.collectionName, + Filter: &qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID), + }, + }, + Limit: &batchSize, + Offset: offset, + WithPayload: qdrant.NewWithPayload(true), + WithVectors: qdrant.NewWithVectors(true), + }) + if err != nil { + log.Errorf("[Qdrant] Failed to query source points: %v", err) + return err + } + + pointsCount := len(scrollResult) + if pointsCount == 0 { + break + } + + log.Infof("[Qdrant] Found %d source points in batch", pointsCount) + + targetPoints := make([]*qdrant.PointStruct, 0, pointsCount) + for _, sourcePoint := range scrollResult { + payload := sourcePoint.Payload + + sourceChunkID := payload[fieldChunkID].GetStringValue() + sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue() + + targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID] + if !ok { + log.Warnf("[Qdrant] Source chunk %s not found in target mapping, skipping", sourceChunkID) + continue + } + + targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceKnowledgeID] + if !ok { + log.Warnf("[Qdrant] Source knowledge %s not found in target mapping, skipping", sourceKnowledgeID) + continue + } + + newPayload := qdrant.NewValueMap(map[string]any{ + fieldContent: payload[fieldContent].GetStringValue(), + fieldSourceID: targetChunkID, + fieldSourceType: payload[fieldSourceType].GetIntegerValue(), + fieldChunkID: targetChunkID, + fieldKnowledgeID: targetKnowledgeID, + fieldKnowledgeBaseID: targetKnowledgeBaseID, + }) + + var vectors *qdrant.Vectors + if vectorOutput := sourcePoint.Vectors.GetVector(); vectorOutput != nil { + if denseVector := vectorOutput.GetDenseVector(); denseVector != nil { + vectors = qdrant.NewVectors(denseVector.Data...) + } + } + + if vectors == nil { + log.Warnf("[Qdrant] No vectors found for source point with chunk %s, skipping", sourceChunkID) + continue + } + + newPoint := &qdrant.PointStruct{ + Id: qdrant.NewID(uuid.New().String()), + Vectors: vectors, + Payload: newPayload, + } + + targetPoints = append(targetPoints, newPoint) + } + + if len(targetPoints) > 0 { + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: q.collectionName, + Points: targetPoints, + }) + if err != nil { + log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err) + return err + } + + totalCopied += len(targetPoints) + log.Infof("[Qdrant] Successfully copied batch, batch size: %d, total copied: %d", + len(targetPoints), totalCopied) + } + + if pointsCount > 0 { + offset = scrollResult[pointsCount-1].Id + } + + if pointsCount < int(batchSize) { + break + } + } + + log.Infof("[Qdrant] Index copy completed, total copied: %d", totalCopied) + return nil +} + +func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value { + payload := map[string]any{ + fieldContent: embedding.Content, + fieldSourceID: embedding.SourceID, + fieldSourceType: int64(embedding.SourceType), + fieldChunkID: embedding.ChunkID, + fieldKnowledgeID: embedding.KnowledgeID, + fieldKnowledgeBaseID: embedding.KnowledgeBaseID, + } + return qdrant.NewValueMap(payload) +} + +func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult { + return []*typesLocal.RetrieveResult{ + { + Results: results, + RetrieverEngineType: typesLocal.QdrantRetrieverEngineType, + RetrieverType: retrieverType, + Error: nil, + }, + } +} + +// Ref: https://github.com/qdrant/qdrant-sizing-calculator +func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding) int64 { + // Payload fields + payloadSizeBytes := int64(0) + payloadSizeBytes += int64(len(embedding.Content)) // content string + payloadSizeBytes += int64(len(embedding.SourceID)) // source_id string + payloadSizeBytes += int64(len(embedding.ChunkID)) // chunk_id string + payloadSizeBytes += int64(len(embedding.KnowledgeID)) // knowledge_id string + payloadSizeBytes += int64(len(embedding.KnowledgeBaseID)) // knowledge_base_id string + payloadSizeBytes += 8 // source_type int64 + + // Vector storage and index + var vectorSizeBytes int64 = 0 + var hnswIndexBytes int64 = 0 + if embedding.Embedding != nil { + dimensions := int64(len(embedding.Embedding)) + vectorSizeBytes = dimensions * 4 + + // HNSW index: dimensions × (M × 2) × 4 bytes + // Default M=16, so: dimensions × 32 × 4 = dimensions × 128 + const hnswM = 16 + hnswIndexBytes = dimensions * (hnswM * 2) * 4 + } + + // ID tracker metadata: 24 bytes per vector + // (forward refs + backward refs + version tracking = 8 + 8 + 8) + const idTrackerBytes int64 = 24 + + totalSizeBytes := payloadSizeBytes + vectorSizeBytes + hnswIndexBytes + idTrackerBytes + return totalSizeBytes +} + +// toQdrantVectorEmbedding converts IndexInfo to Qdrant payload format +func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *QdrantVectorEmbedding { + vector := &QdrantVectorEmbedding{ + Content: embedding.Content, + SourceID: embedding.SourceID, + SourceType: int(embedding.SourceType), + ChunkID: embedding.ChunkID, + KnowledgeID: embedding.KnowledgeID, + KnowledgeBaseID: embedding.KnowledgeBaseID, + } + if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) { + if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok { + vector.Embedding = embeddingMap[embedding.SourceID] + } + } + return vector +} + +// fromQdrantVectorEmbedding converts Qdrant point to IndexWithScore domain model +func fromQdrantVectorEmbedding(id string, + embedding *QdrantVectorEmbeddingWithScore, + matchType types.MatchType, +) *types.IndexWithScore { + return &types.IndexWithScore{ + ID: id, + SourceID: embedding.SourceID, + SourceType: types.SourceType(embedding.SourceType), + ChunkID: embedding.ChunkID, + KnowledgeID: embedding.KnowledgeID, + KnowledgeBaseID: embedding.KnowledgeBaseID, + Content: embedding.Content, + Score: embedding.Score, + MatchType: matchType, + } +} diff --git a/internal/application/repository/retriever/qdrant/structs.go b/internal/application/repository/retriever/qdrant/structs.go new file mode 100644 index 00000000..b2968659 --- /dev/null +++ b/internal/application/repository/retriever/qdrant/structs.go @@ -0,0 +1,23 @@ +package qdrant + +import "github.com/qdrant/go-client/qdrant" + +type qdrantRepository struct { + client *qdrant.Client + collectionName string +} + +type QdrantVectorEmbedding struct { + Content string `json:"content"` + SourceID string `json:"source_id"` + SourceType int `json:"source_type"` + ChunkID string `json:"chunk_id"` + KnowledgeID string `json:"knowledge_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + Embedding []float32 `json:"embedding"` +} + +type QdrantVectorEmbeddingWithScore struct { + QdrantVectorEmbedding + Score float64 +} diff --git a/internal/application/repository/user.go b/internal/application/repository/user.go index 94d0d622..34005a13 100644 --- a/internal/application/repository/user.go +++ b/internal/application/repository/user.go @@ -11,9 +11,9 @@ import ( ) var ( - ErrUserNotFound = errors.New("user not found") + ErrUserNotFound = errors.New("user not found") ErrUserAlreadyExists = errors.New("user already exists") - ErrTokenNotFound = errors.New("token not found") + ErrTokenNotFound = errors.New("token not found") ) // userRepository implements user repository interface @@ -82,15 +82,15 @@ func (r *userRepository) DeleteUser(ctx context.Context, id string) error { func (r *userRepository) ListUsers(ctx context.Context, offset, limit int) ([]*types.User, error) { var users []*types.User query := r.db.WithContext(ctx).Order("created_at DESC") - + if limit > 0 { query = query.Limit(limit) } - + if offset > 0 { query = query.Offset(offset) } - + if err := query.Find(&users).Error; err != nil { return nil, err } @@ -151,4 +151,4 @@ func (r *authTokenRepository) DeleteExpiredTokens(ctx context.Context) error { // RevokeTokensByUserID revokes all tokens for a user func (r *authTokenRepository) RevokeTokensByUserID(ctx context.Context, userID string) error { return r.db.WithContext(ctx).Model(&types.AuthToken{}).Where("user_id = ?", userID).Update("is_revoked", true).Error -} \ No newline at end of file +} diff --git a/internal/container/container.go b/internal/container/container.go index 21590536..7d7c3d9f 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/go-elasticsearch/v8" "github.com/neo4j/neo4j-go-driver/v6/neo4j" "github.com/panjf2000/ants/v2" + "github.com/qdrant/go-client/qdrant" "go.uber.org/dig" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -26,6 +27,7 @@ import ( elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8" neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j" postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres" + qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant" "github.com/Tencent/WeKnora/internal/application/service" chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" "github.com/Tencent/WeKnora/internal/application/service/file" @@ -330,6 +332,54 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret } } } + + if slices.Contains(retrieveDriver, "qdrant") { + qdrantHost := os.Getenv("QDRANT_HOST") + if qdrantHost == "" { + qdrantHost = "localhost" + } + + qdrantPort := 6334 // Default port + if portStr := os.Getenv("QDRANT_PORT"); portStr != "" { + if port, err := strconv.Atoi(portStr); err == nil { + qdrantPort = port + } + } + + // API key for authentication (optional) + qdrantAPIKey := os.Getenv("QDRANT_API_KEY") + + // TLS configuration (optional, defaults to false) + // Enable TLS unless explicitly set to "false" or "0" (case insensitive) + qdrantUseTLS := false + if useTLSStr := os.Getenv("QDRANT_USE_TLS"); useTLSStr != "" { + useTLSLower := strings.ToLower(strings.TrimSpace(useTLSStr)) + qdrantUseTLS = useTLSLower != "false" && useTLSLower != "0" + } + + log.Infof("Connecting to Qdrant at %s:%d (TLS: %v)", qdrantHost, qdrantPort, qdrantUseTLS) + + client, err := qdrant.NewClient(&qdrant.Config{ + Host: qdrantHost, + Port: qdrantPort, + APIKey: qdrantAPIKey, + UseTLS: qdrantUseTLS, + }) + if err != nil { + log.Errorf("Create qdrant client failed: %v", err) + } else { + qdrantRepository := qdrantRepo.NewQdrantRetrieveEngineRepository(client) + if err := registry.Register( + retriever.NewKVHybridRetrieveEngine( + qdrantRepository, types.QdrantRetrieverEngineType, + ), + ); err != nil { + log.Errorf("Register qdrant retrieve engine failed: %v", err) + } else { + log.Infof("Register qdrant retrieve engine success") + } + } + } return registry, nil } diff --git a/internal/types/retriever.go b/internal/types/retriever.go index e4de533c..67ff9a51 100644 --- a/internal/types/retriever.go +++ b/internal/types/retriever.go @@ -9,6 +9,7 @@ const ( ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch" InfinityRetrieverEngineType RetrieverEngineType = "infinity" ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss" + QdrantRetrieverEngineType RetrieverEngineType = "qdrant" ) // RetrieverType represents the type of retriever