Merge remote-tracking branch 'github_public/main'
This commit is contained in:
+17
-1
@@ -14,7 +14,7 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
# 主数据库类型(postgres/mysql)
|
||||
DB_DRIVER=postgres
|
||||
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8)
|
||||
# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8/qdrant)
|
||||
RETRIEVE_DRIVER=postgres
|
||||
|
||||
# 文件存储类型(local/minio/cos)
|
||||
@@ -85,6 +85,22 @@ CONCURRENCY_POOL_SIZE=5
|
||||
# ElasticSearch索引名称,用于存储向量数据
|
||||
# ELASTICSEARCH_INDEX=WeKnora
|
||||
|
||||
# 如果使用Qdrant作为向量存储,需要配置以下参数
|
||||
# Qdrant服务主机地址
|
||||
# QDRANT_HOST=localhost
|
||||
|
||||
# Qdrant服务端口
|
||||
# QDRANT_PORT=6334
|
||||
|
||||
# Qdrant集合名称,用于存储向量数据
|
||||
# QDRANT_COLLECTION=weknora_embeddings
|
||||
|
||||
# Qdrant API密钥,如果需要身份验证(可选)
|
||||
# QDRANT_API_KEY=your_qdrant_api_key
|
||||
|
||||
# 是否启用TLS加密连接(可选,默认为false)
|
||||
# QDRANT_USE_TLS=false
|
||||
|
||||
# 如果使用MinIO作为文件存储,需要配置以下参数
|
||||
# MinIO访问密钥
|
||||
# MINIO_ACCESS_KEY_ID=your_minio_access_key
|
||||
|
||||
+77
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [0.2.0] - 2025-11-26
|
||||
## [0.2.0] - 2025-12-05
|
||||
|
||||
### 🚀 Major Features
|
||||
- **NEW**: ReACT Agent Mode
|
||||
@@ -81,6 +81,82 @@ All notable changes to this project will be documented in this file.
|
||||
- Updated README files with v0.2.0 highlights (English, Chinese, Japanese)
|
||||
- Added latest updates section in all README files
|
||||
- Updated architecture diagrams and feature matrices
|
||||
## [0.1.6] - 2025-11-24
|
||||
|
||||
### Document Parser Enhancements
|
||||
- NEW: Added CSV, XLSX, XLS file parsing support (spreadsheet processing, tabular data extraction)
|
||||
- NEW: Web page parser (dedicated class, optimized web image encoding, improved dependency management)
|
||||
|
||||
### Document Processing Improvements
|
||||
- NEW: MarkdownTableUtil (reduced whitespace, improved table readability/consistency)
|
||||
- NEW: Document model class (structured models for type safety, optimized config/parsing logic)
|
||||
- UPGRADED: Docx2Parser (enhanced timeout handling, better image processing, optimized OCR backend)
|
||||
|
||||
### Internationalization
|
||||
- NEW: English/Russian multi-language support (vue-i18n integration, translated UI/text/errors, multilingual docs for knowledge graph/MCP config)
|
||||
|
||||
### Bug Fixes
|
||||
- Fixed menu component integration issues
|
||||
- Fixed Darwin (macOS) memory check regex error (resolved empty output)
|
||||
- Fixed model availability check (unified logic, auto ":latest" tag, prevented duplicate pull calls)
|
||||
- Fixed Docker Compose security vulnerability (addressed writable filesystem issue)
|
||||
|
||||
### Refactoring & Optimization
|
||||
- Refactored parser logging/API checks (simplified exception handling, better error reporting)
|
||||
- Refactored chunk processing (removed redundant header handling, updated examples)
|
||||
- Refactored module organization (docreader structure, proto/client imports, Docker config, absolute imports)
|
||||
|
||||
### Documentation Updates
|
||||
- Updated API Key acquisition docs (web registration + account page retrieval)
|
||||
- Updated Docker Compose setup guide (comprehensive instructions, config adjustments)
|
||||
- Updated multilingual docs (added knowledge graph/MCP config guides, directory structure)
|
||||
- Removed deprecated hybrid search API docs
|
||||
|
||||
### Code Cleanup
|
||||
- Removed redundant Docker build parameters
|
||||
- Updated .gitignore rules
|
||||
- Optimized import statements/type hints
|
||||
- Cleaned redundant logging/comments
|
||||
|
||||
### CI/CD Improvements
|
||||
- Added new CI/CD trigger branches
|
||||
- Added build concurrency control
|
||||
- Added disk space cleanup automation
|
||||
|
||||
## [0.1.5] - 2025-10-20
|
||||
|
||||
### Features & Enhancements
|
||||
- Added multi-knowledgebases operation support and management (UI & backend logic)
|
||||
- Enhanced tenant information management: New tenant page with user-friendly storage quota and usage rate display (see TenantInfo.vue)
|
||||
- Initialization Wizard improvements: Stricter form validation, VLM/OpenAI compatible URL verification, and multimodal file upload preview & validation (see InitializationContent.vue)
|
||||
- Backend: API Key automatic generation and update logic (see types.Tenant & tenantService.UpdateTenant)
|
||||
|
||||
### UI / UX
|
||||
- Restructured settings page and initialization page layouts; optimized button states, loading states, and prompt messages; improved upload/preview experience
|
||||
- Enhanced menu component: Multi-knowledgebase switching and pre-upload validation logic (see menu.vue)
|
||||
- Hidden/protected sensitive information (e.g., API Keys) and added copy interaction prompts (see TenantInfo.vue)
|
||||
|
||||
### Security Fixes
|
||||
- Fixed potential frontend XSS vulnerabilities; enhanced input validation and Content Security Policy
|
||||
- Hidden API Keys in UI and improved copy behavior prompts to strengthen information leakage protection
|
||||
|
||||
### Bug Fixes
|
||||
- Resolved OCR/AVX support-related issues and image parsing concurrency errors
|
||||
- Fixed frontend routing/login redirection issues and file download content errors
|
||||
- Fixed docreader service health check and model prefetching issues
|
||||
|
||||
### DevOps / Building
|
||||
- Improved image building scripts: Enhanced platform/architecture detection (amd64 / arm64) and injected version information during build (see get_version.sh & build_images.sh)
|
||||
- Refined Makefile and build process to facilitate CI injection of LDFLAGS (see Makefile)
|
||||
- Improved usage and documentation for scripts and migration tools (migrate) (see migrate.sh)
|
||||
|
||||
### Documentation
|
||||
- Updated README and multilingual documentation (EN/CN/JA) along with release/CHANGELOG (see CHANGELOG.md & README.md for details)
|
||||
- Added MCP server usage instructions and installation guide (see mcp-server/INSTALL.md)
|
||||
|
||||
### Developer / Internal API Changes (For Reference)
|
||||
- New/updated backend system information response structure: handler.GetSystemInfoResponse
|
||||
- Tenant data structure and JSON storage fields: types.Tenant
|
||||
|
||||
## [0.1.4] - 2025-09-17
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ require (
|
||||
github.com/panjf2000/ants/v2 v2.11.2
|
||||
github.com/parquet-go/parquet-go v0.25.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/qdrant/go-client v1.16.1
|
||||
github.com/redis/go-redis/v9 v9.14.0
|
||||
github.com/sashabaranov/go-openai v1.40.5
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
@@ -30,17 +31,17 @@ require (
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/tencentyun/cos-go-sdk-v5 v0.7.65
|
||||
github.com/yanyiwu/gojieba v1.4.5
|
||||
go.opentelemetry.io/otel v1.37.0
|
||||
go.opentelemetry.io/otel v1.38.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0
|
||||
go.opentelemetry.io/otel/sdk v1.37.0
|
||||
go.opentelemetry.io/otel/trace v1.37.0
|
||||
go.opentelemetry.io/otel/trace v1.38.0
|
||||
go.uber.org/dig v1.18.1
|
||||
golang.org/x/crypto v0.43.0
|
||||
golang.org/x/sync v0.17.0
|
||||
google.golang.org/grpc v1.73.0
|
||||
google.golang.org/protobuf v1.36.9
|
||||
golang.org/x/crypto v0.44.0
|
||||
golang.org/x/sync v0.18.0
|
||||
google.golang.org/grpc v1.76.0
|
||||
google.golang.org/protobuf v1.36.10
|
||||
gorm.io/driver/postgres v1.5.11
|
||||
gorm.io/gorm v1.25.12
|
||||
)
|
||||
@@ -88,7 +89,7 @@ require (
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
@@ -118,16 +119,16 @@ require (
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.37.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/net v0.46.0 // indirect
|
||||
golang.org/x/sys v0.37.0 // indirect
|
||||
golang.org/x/text v0.30.0 // indirect
|
||||
golang.org/x/time v0.13.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
@@ -52,10 +52,10 @@ github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
|
||||
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
|
||||
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
@@ -111,8 +111,6 @@ github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
@@ -160,8 +158,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
|
||||
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
@@ -196,8 +194,8 @@ github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGg
|
||||
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
||||
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
|
||||
github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -215,8 +213,8 @@ github.com/ollama/ollama v0.11.4 h1:6xLYLEPTKtw6N20qQecyEL/rrBktPO4o5U05cnvkSmI=
|
||||
github.com/ollama/ollama v0.11.4/go.mod h1:9+1//yWPsDE2u+l1a5mpaKrYw4VdnSsRU3ioq5BvMms=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
|
||||
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/panjf2000/ants/v2 v2.11.2 h1:AVGpMSePxUNpcLaBO34xuIgM1ZdKOiGnpxLXixLi5Jo=
|
||||
@@ -233,6 +231,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/qdrant/go-client v1.16.1 h1:Jr47kz0k8I+U2sUm2UUO2eq2kL0fTcgjLPIz6a0RKuQ=
|
||||
github.com/qdrant/go-client v1.16.1/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw=
|
||||
github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE=
|
||||
github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
@@ -240,8 +240,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
|
||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
|
||||
@@ -308,26 +308,26 @@ github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgTNQVet8iTDW7oIk03tXHq+wkwIDnE=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
|
||||
go.uber.org/dig v1.18.1 h1:rLww6NuajVjeQn+49u5NcezUJEGwd5uXmyoCKW2g5Es=
|
||||
@@ -344,8 +344,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
|
||||
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
|
||||
golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
|
||||
golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
@@ -360,8 +360,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
|
||||
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -369,8 +369,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -384,8 +384,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -395,8 +395,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
|
||||
golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
@@ -406,10 +406,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
|
||||
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||
golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
|
||||
golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
@@ -418,14 +418,16 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
|
||||
google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok=
|
||||
google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc=
|
||||
google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
|
||||
google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:oDOGiMSXHL4sDTJvFvIB9nRQCGdLP1o/iVaqQK8zB+M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
|
||||
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@@ -0,0 +1,572 @@
|
||||
package qdrant
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
typesLocal "github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
const (
|
||||
envQdrantCollection = "QDRANT_COLLECTION"
|
||||
defaultCollectionName = "weknora_embeddings"
|
||||
fieldContent = "content"
|
||||
fieldSourceID = "source_id"
|
||||
fieldSourceType = "source_type"
|
||||
fieldChunkID = "chunk_id"
|
||||
fieldKnowledgeID = "knowledge_id"
|
||||
fieldKnowledgeBaseID = "knowledge_base_id"
|
||||
fieldEmbedding = "embedding"
|
||||
)
|
||||
|
||||
// NewQdrantRetrieveEngineRepository creates and initializes a new Qdrant repository
|
||||
func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.RetrieveEngineRepository {
|
||||
log := logger.GetLogger(context.Background())
|
||||
log.Info("[Qdrant] Initializing Qdrant retriever engine repository")
|
||||
|
||||
collectionName := os.Getenv(envQdrantCollection)
|
||||
if collectionName == "" {
|
||||
log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name")
|
||||
collectionName = defaultCollectionName
|
||||
}
|
||||
|
||||
res := &qdrantRepository{
|
||||
client: client,
|
||||
collectionName: collectionName,
|
||||
}
|
||||
|
||||
log.Info("[Qdrant] Successfully initialized repository")
|
||||
return res
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType {
|
||||
return typesLocal.QdrantRetrieverEngineType
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) Support() []typesLocal.RetrieverType {
|
||||
return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType}
|
||||
}
|
||||
|
||||
// EstimateStorageSize calculates the estimated storage size for a list of indices
|
||||
func (q *qdrantRepository) EstimateStorageSize(ctx context.Context,
|
||||
indexInfoList []*typesLocal.IndexInfo, params map[string]any,
|
||||
) int64 {
|
||||
var totalStorageSize int64
|
||||
for _, embedding := range indexInfoList {
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, params)
|
||||
totalStorageSize += q.calculateStorageSize(embeddingDB)
|
||||
}
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"[Qdrant] Storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize,
|
||||
)
|
||||
return totalStorageSize
|
||||
}
|
||||
|
||||
// Save stores a single point in Qdrant
|
||||
func (q *qdrantRepository) Save(ctx context.Context,
|
||||
embedding *typesLocal.IndexInfo,
|
||||
additionalParams map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Debugf("[Qdrant] Saving index for chunk ID: %s", embedding.ChunkID)
|
||||
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
|
||||
if len(embeddingDB.Embedding) == 0 {
|
||||
err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID)
|
||||
log.Errorf("[Qdrant] %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
pointID := uuid.New().String()
|
||||
point := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(pointID),
|
||||
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
|
||||
Payload: createPayload(embeddingDB),
|
||||
}
|
||||
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: []*qdrant.PointStruct{point},
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to save index: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchSave stores multiple points in Qdrant using batch upsert
|
||||
func (q *qdrantRepository) BatchSave(ctx context.Context,
|
||||
embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(embeddingList) == 0 {
|
||||
log.Warn("[Qdrant] Empty list provided to BatchSave, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList))
|
||||
|
||||
points := make([]*qdrant.PointStruct, 0, len(embeddingList))
|
||||
for _, embedding := range embeddingList {
|
||||
embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams)
|
||||
if len(embeddingDB.Embedding) == 0 {
|
||||
log.Warnf("[Qdrant] Skipping empty embedding for chunk ID: %s", embedding.ChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
point := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: qdrant.NewVectors(embeddingDB.Embedding...),
|
||||
Payload: createPayload(embeddingDB),
|
||||
}
|
||||
points = append(points, point)
|
||||
log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID)
|
||||
}
|
||||
|
||||
if len(points) == 0 {
|
||||
log.Warn("[Qdrant] No valid points to save after filtering")
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: points,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to execute batch operation: %v", err)
|
||||
return fmt.Errorf("failed to batch save: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully batch saved %d indices", len(points))
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList removes points from the collection based on chunk IDs
|
||||
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkIDList) == 0 {
|
||||
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
|
||||
|
||||
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to delete by chunk IDs: %v", err)
|
||||
return fmt.Errorf("failed to delete by chunk IDs: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully deleted documents by chunk IDs")
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
|
||||
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(knowledgeIDList) == 0 {
|
||||
log.Warn("[Qdrant] Empty knowledge ID list provided for deletion, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
|
||||
|
||||
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to delete by knowledge IDs: %v", err)
|
||||
return fmt.Errorf("failed to delete by knowledge IDs: %w", err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully deleted documents by knowledge IDs")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter {
|
||||
must := make([]*qdrant.Condition, 0)
|
||||
mustNot := make([]*qdrant.Condition, 0)
|
||||
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeChunkIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldChunkID, params.ExcludeChunkIDs...))
|
||||
}
|
||||
|
||||
return &qdrant.Filter{
|
||||
Must: must,
|
||||
MustNot: mustNot,
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type
|
||||
func (q *qdrantRepository) Retrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType)
|
||||
|
||||
switch params.RetrieverType {
|
||||
case typesLocal.VectorRetrieverType:
|
||||
return q.VectorRetrieve(ctx, params)
|
||||
case typesLocal.KeywordsRetrieverType:
|
||||
return q.KeywordsRetrieve(ctx, params)
|
||||
}
|
||||
|
||||
err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType)
|
||||
log.Errorf("[Qdrant] %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VectorRetrieve performs vector similarity search
|
||||
func (q *qdrantRepository) VectorRetrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
|
||||
len(params.Embedding), params.TopK, params.Threshold)
|
||||
|
||||
filter := q.getBaseFilter(params)
|
||||
|
||||
limit := uint64(params.TopK)
|
||||
scoreThreshold := float32(params.Threshold)
|
||||
|
||||
searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Query: qdrant.NewQuery(params.Embedding...),
|
||||
Filter: filter,
|
||||
Limit: &limit,
|
||||
ScoreThreshold: &scoreThreshold,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Vector search failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*typesLocal.IndexWithScore
|
||||
for _, point := range searchResult {
|
||||
payload := point.Payload
|
||||
embedding := &QdrantVectorEmbeddingWithScore{
|
||||
QdrantVectorEmbedding: QdrantVectorEmbedding{
|
||||
Content: payload[fieldContent].GetStringValue(),
|
||||
SourceID: payload[fieldSourceID].GetStringValue(),
|
||||
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
},
|
||||
Score: float64(point.Score),
|
||||
}
|
||||
|
||||
pointID := point.Id.GetUuid()
|
||||
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding))
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
log.Warnf("[Qdrant] No vector matches found that meet threshold %.4f", params.Threshold)
|
||||
} else {
|
||||
log.Infof("[Qdrant] Vector retrieval found %d results", len(results))
|
||||
log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score)
|
||||
}
|
||||
|
||||
return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil
|
||||
}
|
||||
|
||||
// KeywordsRetrieve performs keyword-based search in document content
|
||||
func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context,
|
||||
params typesLocal.RetrieveParams,
|
||||
) ([]*typesLocal.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK)
|
||||
|
||||
filter := q.getBaseFilter(params)
|
||||
|
||||
filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query))
|
||||
|
||||
limit := uint32(params.TopK)
|
||||
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Filter: filter,
|
||||
Limit: &limit,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Keywords search failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*typesLocal.IndexWithScore
|
||||
for _, point := range scrollResult {
|
||||
payload := point.Payload
|
||||
embedding := &QdrantVectorEmbeddingWithScore{
|
||||
QdrantVectorEmbedding: QdrantVectorEmbedding{
|
||||
Content: payload[fieldContent].GetStringValue(),
|
||||
SourceID: payload[fieldSourceID].GetStringValue(),
|
||||
SourceType: int(payload[fieldSourceType].GetIntegerValue()),
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
},
|
||||
Score: 1.0,
|
||||
}
|
||||
|
||||
pointID := point.Id.GetUuid()
|
||||
results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords))
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query)
|
||||
} else {
|
||||
log.Infof("[Qdrant] Keywords retrieval found %d results", len(results))
|
||||
}
|
||||
|
||||
return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
// CopyIndices copies index data from source knowledge base to target knowledge base
|
||||
func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
sourceKnowledgeBaseID string,
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof(
|
||||
"[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d",
|
||||
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
|
||||
)
|
||||
|
||||
if len(sourceToTargetChunkIDMap) == 0 {
|
||||
log.Warn("[Qdrant] Empty mapping, skipping copy")
|
||||
return nil
|
||||
}
|
||||
|
||||
batchSize := uint32(64)
|
||||
var offset *qdrant.PointId = nil
|
||||
totalCopied := 0
|
||||
|
||||
for {
|
||||
scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Filter: &qdrant.Filter{
|
||||
Must: []*qdrant.Condition{
|
||||
qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID),
|
||||
},
|
||||
},
|
||||
Limit: &batchSize,
|
||||
Offset: offset,
|
||||
WithPayload: qdrant.NewWithPayload(true),
|
||||
WithVectors: qdrant.NewWithVectors(true),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to query source points: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
pointsCount := len(scrollResult)
|
||||
if pointsCount == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Found %d source points in batch", pointsCount)
|
||||
|
||||
targetPoints := make([]*qdrant.PointStruct, 0, pointsCount)
|
||||
for _, sourcePoint := range scrollResult {
|
||||
payload := sourcePoint.Payload
|
||||
|
||||
sourceChunkID := payload[fieldChunkID].GetStringValue()
|
||||
sourceKnowledgeID := payload[fieldKnowledgeID].GetStringValue()
|
||||
|
||||
targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID]
|
||||
if !ok {
|
||||
log.Warnf("[Qdrant] Source chunk %s not found in target mapping, skipping", sourceChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceKnowledgeID]
|
||||
if !ok {
|
||||
log.Warnf("[Qdrant] Source knowledge %s not found in target mapping, skipping", sourceKnowledgeID)
|
||||
continue
|
||||
}
|
||||
|
||||
newPayload := qdrant.NewValueMap(map[string]any{
|
||||
fieldContent: payload[fieldContent].GetStringValue(),
|
||||
fieldSourceID: targetChunkID,
|
||||
fieldSourceType: payload[fieldSourceType].GetIntegerValue(),
|
||||
fieldChunkID: targetChunkID,
|
||||
fieldKnowledgeID: targetKnowledgeID,
|
||||
fieldKnowledgeBaseID: targetKnowledgeBaseID,
|
||||
})
|
||||
|
||||
var vectors *qdrant.Vectors
|
||||
if vectorOutput := sourcePoint.Vectors.GetVector(); vectorOutput != nil {
|
||||
if denseVector := vectorOutput.GetDenseVector(); denseVector != nil {
|
||||
vectors = qdrant.NewVectors(denseVector.Data...)
|
||||
}
|
||||
}
|
||||
|
||||
if vectors == nil {
|
||||
log.Warnf("[Qdrant] No vectors found for source point with chunk %s, skipping", sourceChunkID)
|
||||
continue
|
||||
}
|
||||
|
||||
newPoint := &qdrant.PointStruct{
|
||||
Id: qdrant.NewID(uuid.New().String()),
|
||||
Vectors: vectors,
|
||||
Payload: newPayload,
|
||||
}
|
||||
|
||||
targetPoints = append(targetPoints, newPoint)
|
||||
}
|
||||
|
||||
if len(targetPoints) > 0 {
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: q.collectionName,
|
||||
Points: targetPoints,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
totalCopied += len(targetPoints)
|
||||
log.Infof("[Qdrant] Successfully copied batch, batch size: %d, total copied: %d",
|
||||
len(targetPoints), totalCopied)
|
||||
}
|
||||
|
||||
if pointsCount > 0 {
|
||||
offset = scrollResult[pointsCount-1].Id
|
||||
}
|
||||
|
||||
if pointsCount < int(batchSize) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Index copy completed, total copied: %d", totalCopied)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value {
|
||||
payload := map[string]any{
|
||||
fieldContent: embedding.Content,
|
||||
fieldSourceID: embedding.SourceID,
|
||||
fieldSourceType: int64(embedding.SourceType),
|
||||
fieldChunkID: embedding.ChunkID,
|
||||
fieldKnowledgeID: embedding.KnowledgeID,
|
||||
fieldKnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
}
|
||||
return qdrant.NewValueMap(payload)
|
||||
}
|
||||
|
||||
func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult {
|
||||
return []*typesLocal.RetrieveResult{
|
||||
{
|
||||
Results: results,
|
||||
RetrieverEngineType: typesLocal.QdrantRetrieverEngineType,
|
||||
RetrieverType: retrieverType,
|
||||
Error: nil,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Ref: https://github.com/qdrant/qdrant-sizing-calculator
|
||||
func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding) int64 {
|
||||
// Payload fields
|
||||
payloadSizeBytes := int64(0)
|
||||
payloadSizeBytes += int64(len(embedding.Content)) // content string
|
||||
payloadSizeBytes += int64(len(embedding.SourceID)) // source_id string
|
||||
payloadSizeBytes += int64(len(embedding.ChunkID)) // chunk_id string
|
||||
payloadSizeBytes += int64(len(embedding.KnowledgeID)) // knowledge_id string
|
||||
payloadSizeBytes += int64(len(embedding.KnowledgeBaseID)) // knowledge_base_id string
|
||||
payloadSizeBytes += 8 // source_type int64
|
||||
|
||||
// Vector storage and index
|
||||
var vectorSizeBytes int64 = 0
|
||||
var hnswIndexBytes int64 = 0
|
||||
if embedding.Embedding != nil {
|
||||
dimensions := int64(len(embedding.Embedding))
|
||||
vectorSizeBytes = dimensions * 4
|
||||
|
||||
// HNSW index: dimensions × (M × 2) × 4 bytes
|
||||
// Default M=16, so: dimensions × 32 × 4 = dimensions × 128
|
||||
const hnswM = 16
|
||||
hnswIndexBytes = dimensions * (hnswM * 2) * 4
|
||||
}
|
||||
|
||||
// ID tracker metadata: 24 bytes per vector
|
||||
// (forward refs + backward refs + version tracking = 8 + 8 + 8)
|
||||
const idTrackerBytes int64 = 24
|
||||
|
||||
totalSizeBytes := payloadSizeBytes + vectorSizeBytes + hnswIndexBytes + idTrackerBytes
|
||||
return totalSizeBytes
|
||||
}
|
||||
|
||||
// toQdrantVectorEmbedding converts IndexInfo to Qdrant payload format
|
||||
func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *QdrantVectorEmbedding {
|
||||
vector := &QdrantVectorEmbedding{
|
||||
Content: embedding.Content,
|
||||
SourceID: embedding.SourceID,
|
||||
SourceType: int(embedding.SourceType),
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
}
|
||||
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
|
||||
if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok {
|
||||
vector.Embedding = embeddingMap[embedding.SourceID]
|
||||
}
|
||||
}
|
||||
return vector
|
||||
}
|
||||
|
||||
// fromQdrantVectorEmbedding converts Qdrant point to IndexWithScore domain model
|
||||
func fromQdrantVectorEmbedding(id string,
|
||||
embedding *QdrantVectorEmbeddingWithScore,
|
||||
matchType types.MatchType,
|
||||
) *types.IndexWithScore {
|
||||
return &types.IndexWithScore{
|
||||
ID: id,
|
||||
SourceID: embedding.SourceID,
|
||||
SourceType: types.SourceType(embedding.SourceType),
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
Content: embedding.Content,
|
||||
Score: embedding.Score,
|
||||
MatchType: matchType,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package qdrant
|
||||
|
||||
import "github.com/qdrant/go-client/qdrant"
|
||||
|
||||
type qdrantRepository struct {
|
||||
client *qdrant.Client
|
||||
collectionName string
|
||||
}
|
||||
|
||||
type QdrantVectorEmbedding struct {
|
||||
Content string `json:"content"`
|
||||
SourceID string `json:"source_id"`
|
||||
SourceType int `json:"source_type"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
}
|
||||
|
||||
type QdrantVectorEmbeddingWithScore struct {
|
||||
QdrantVectorEmbedding
|
||||
Score float64
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
"github.com/panjf2000/ants/v2"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.uber.org/dig"
|
||||
"gorm.io/driver/postgres"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
@@ -421,6 +423,54 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if slices.Contains(retrieveDriver, "qdrant") {
|
||||
qdrantHost := os.Getenv("QDRANT_HOST")
|
||||
if qdrantHost == "" {
|
||||
qdrantHost = "localhost"
|
||||
}
|
||||
|
||||
qdrantPort := 6334 // Default port
|
||||
if portStr := os.Getenv("QDRANT_PORT"); portStr != "" {
|
||||
if port, err := strconv.Atoi(portStr); err == nil {
|
||||
qdrantPort = port
|
||||
}
|
||||
}
|
||||
|
||||
// API key for authentication (optional)
|
||||
qdrantAPIKey := os.Getenv("QDRANT_API_KEY")
|
||||
|
||||
// TLS configuration (optional, defaults to false)
|
||||
// Enable TLS unless explicitly set to "false" or "0" (case insensitive)
|
||||
qdrantUseTLS := false
|
||||
if useTLSStr := os.Getenv("QDRANT_USE_TLS"); useTLSStr != "" {
|
||||
useTLSLower := strings.ToLower(strings.TrimSpace(useTLSStr))
|
||||
qdrantUseTLS = useTLSLower != "false" && useTLSLower != "0"
|
||||
}
|
||||
|
||||
log.Infof("Connecting to Qdrant at %s:%d (TLS: %v)", qdrantHost, qdrantPort, qdrantUseTLS)
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: qdrantHost,
|
||||
Port: qdrantPort,
|
||||
APIKey: qdrantAPIKey,
|
||||
UseTLS: qdrantUseTLS,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("Create qdrant client failed: %v", err)
|
||||
} else {
|
||||
qdrantRepository := qdrantRepo.NewQdrantRetrieveEngineRepository(client)
|
||||
if err := registry.Register(
|
||||
retriever.NewKVHybridRetrieveEngine(
|
||||
qdrantRepository, types.QdrantRetrieverEngineType,
|
||||
),
|
||||
); err != nil {
|
||||
log.Errorf("Register qdrant retrieve engine failed: %v", err)
|
||||
} else {
|
||||
log.Infof("Register qdrant retrieve engine success")
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ const (
|
||||
ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch"
|
||||
InfinityRetrieverEngineType RetrieverEngineType = "infinity"
|
||||
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
|
||||
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
|
||||
)
|
||||
|
||||
// RetrieverType represents the type of retriever
|
||||
|
||||
Reference in New Issue
Block a user