Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: update storage size calculations for vector embeddings

Browse Source 
- Revised the storage size calculation logic for Milvus, Qdrant, and Weaviate repositories to provide more accurate estimates based on the actual data structure and indexing methods.
- Updated comments to clarify the relationship between vector dimensions and index storage requirements, enhancing code readability and maintainability.
This commit is contained in:
wizardchen
2026-03-09 12:13:49 +08:00
committed by lyingbug
parent dd39fbc018
commit 33ccc41bea
3 changed files with 13 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/application/repository/retriever/milvus/repository.go
+5 -4
View File
@@ -901,10 +901,11 @@ func (m *milvusRepository) calculateStorageSize(embedding *MilvusVectorEmbedding
dimensions := int64(len(embedding.Embedding))
vectorSizeBytes = dimensions * 4
// IVF_FLAT index: dimensions × (nlist × 4 + 4) bytes
// Default nlist=16384, so: dimensions × (65536 + 4) ≈ dimensions × 65540
const nlist = 16384
indexBytes = dimensions * (nlist*4 + 4)
// IVF_FLAT index: vectors are duplicated in inverted lists (grouped by cluster),
// plus a small per-vector overhead for cluster assignment and list management.
// The centroid table (nlist × dim × 4) is a shared structure and should NOT
// be counted per-vector.
indexBytes = vectorSizeBytes + 16
}
// ID tracker and metadata: ~32 bytes per vector
internal/application/repository/retriever/qdrant/repository.go
+4 -3
View File
@@ -904,10 +904,11 @@ func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding
dimensions := int64(len(embedding.Embedding))
vectorSizeBytes = dimensions * 4
// HNSW index: dimensions × (M × 2) × 4 bytes
// Default M=16, so: dimensions × 32 × 4 = dimensions × 128
// HNSW graph links per vector: M×2 neighbors in layer 0, ~8 bytes per link
// (4 bytes PointOffsetType + multi-layer amortization).
// Graph link count depends on M, NOT on vector dimensions.
const hnswM = 16
hnswIndexBytes = dimensions * (hnswM * 2) * 4
hnswIndexBytes = hnswM * 2 * 8
}
// ID tracker metadata: 24 bytes per vector
internal/application/repository/retriever/weaviate/repository.go
+4 -3
View File
@@ -971,10 +971,11 @@ func (w *weaviateRepository) calculateStorageSize(embedding *WeaviateVectorEmbed
dimensions := int64(len(embedding.Embedding))
vectorSizeBytes = dimensions * 4
// HNSW index: dimensions × (M × 2) × 4 bytes
// Default M=32, so: dimensions × 64 × 4 = dimensions × 256
// HNSW graph links per vector: M×2 neighbors in layer 0, ~8 bytes per link
// (4 bytes for neighbor ID + multi-layer amortization).
// Graph link count depends on M, NOT on vector dimensions.
const hnswM = 32
hnswIndexBytes = dimensions * (hnswM * 2) * 4
hnswIndexBytes = hnswM * 2 * 8
}
// ID tracker metadata: 24 bytes per vector