diff --git a/base/search/ivf.go b/base/search/ivf.go
index f362e3f14..4ddd784b8 100644
--- a/base/search/ivf.go
+++ b/base/search/ivf.go
@@ -215,6 +215,9 @@ func (idx *IVF) Build(_ context.Context) {
 						nextDistance = d
 					}
 				}
+				if nextCluster == -1 {
+					return nil
+				}
 				if nextCluster != assignments[i] {
 					errorCount.Inc()
 				}