From 33ccc41bea6c7ea7ad7284746aa5a4e843333652 Mon Sep 17 00:00:00 2001
From: wizardchen <wizardchen@tencent.com>
Date: Mon, 9 Mar 2026 12:13:49 +0800
Subject: [PATCH] refactor: update storage size calculations for vector
 embeddings

- Revised the storage size calculation logic for Milvus, Qdrant, and Weaviate repositories to provide more accurate estimates based on the actual data structure and indexing methods.
- Updated comments to clarify the relationship between vector dimensions and index storage requirements, enhancing code readability and maintainability.
---
 .../repository/retriever/milvus/repository.go            | 9 +++++----
 .../repository/retriever/qdrant/repository.go            | 7 ++++---
 .../repository/retriever/weaviate/repository.go          | 7 ++++---
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/internal/application/repository/retriever/milvus/repository.go b/internal/application/repository/retriever/milvus/repository.go
index 3f8d8137..fdbe357a 100644
--- a/internal/application/repository/retriever/milvus/repository.go
+++ b/internal/application/repository/retriever/milvus/repository.go
@@ -901,10 +901,11 @@ func (m *milvusRepository) calculateStorageSize(embedding *MilvusVectorEmbedding
 		dimensions := int64(len(embedding.Embedding))
 		vectorSizeBytes = dimensions * 4
 
-		// IVF_FLAT index: dimensions × (nlist × 4 + 4) bytes
-		// Default nlist=16384, so: dimensions × (65536 + 4) ≈ dimensions × 65540
-		const nlist = 16384
-		indexBytes = dimensions * (nlist*4 + 4)
+		// IVF_FLAT index: vectors are duplicated in inverted lists (grouped by cluster),
+		// plus a small per-vector overhead for cluster assignment and list management.
+		// The centroid table (nlist × dim × 4) is a shared structure and should NOT
+		// be counted per-vector.
+		indexBytes = vectorSizeBytes + 16
 	}
 
 	// ID tracker and metadata: ~32 bytes per vector
diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go
index dfbbb575..29e74f5c 100644
--- a/internal/application/repository/retriever/qdrant/repository.go
+++ b/internal/application/repository/retriever/qdrant/repository.go
@@ -904,10 +904,11 @@ func (q *qdrantRepository) calculateStorageSize(embedding *QdrantVectorEmbedding
 		dimensions := int64(len(embedding.Embedding))
 		vectorSizeBytes = dimensions * 4
 
-		// HNSW index: dimensions × (M × 2) × 4 bytes
-		// Default M=16, so: dimensions × 32 × 4 = dimensions × 128
+		// HNSW graph links per vector: M×2 neighbors in layer 0, ~8 bytes per link
+		// (4 bytes PointOffsetType + multi-layer amortization).
+		// Graph link count depends on M, NOT on vector dimensions.
 		const hnswM = 16
-		hnswIndexBytes = dimensions * (hnswM * 2) * 4
+		hnswIndexBytes = hnswM * 2 * 8
 	}
 
 	// ID tracker metadata: 24 bytes per vector
diff --git a/internal/application/repository/retriever/weaviate/repository.go b/internal/application/repository/retriever/weaviate/repository.go
index 1fbfb2f6..fbdc8e72 100644
--- a/internal/application/repository/retriever/weaviate/repository.go
+++ b/internal/application/repository/retriever/weaviate/repository.go
@@ -971,10 +971,11 @@ func (w *weaviateRepository) calculateStorageSize(embedding *WeaviateVectorEmbed
 		dimensions := int64(len(embedding.Embedding))
 		vectorSizeBytes = dimensions * 4
 
-		// HNSW index: dimensions × (M × 2) × 4 bytes
-		// Default M=32, so: dimensions × 64 × 4 = dimensions × 256
+		// HNSW graph links per vector: M×2 neighbors in layer 0, ~8 bytes per link
+		// (4 bytes for neighbor ID + multi-layer amortization).
+		// Graph link count depends on M, NOT on vector dimensions.
 		const hnswM = 32
-		hnswIndexBytes = dimensions * (hnswM * 2) * 4
+		hnswIndexBytes = hnswM * 2 * 8
 	}
 
 	// ID tracker metadata: 24 bytes per vector