diff --git a/python/cuml/test/test_nearest_neighbors.py b/python/cuml/test/test_nearest_neighbors.py
index 32b0499d8d..4bb21b9870 100644
--- a/python/cuml/test/test_nearest_neighbors.py
+++ b/python/cuml/test/test_nearest_neighbors.py
@@ -513,11 +513,15 @@ def test_knn_graph(input_type, mode, output_type, as_instance,
 
 
 @pytest.mark.parametrize('distance', ["euclidean", "haversine"])
-@pytest.mark.parametrize('n_neighbors', [2, 12])
-@pytest.mark.parametrize('nrows', [unit_param(1000), stress_param(70000)])
+@pytest.mark.parametrize('n_neighbors', [4, 25])
+@pytest.mark.parametrize('nrows', [unit_param(10000), stress_param(70000)])
 def test_nearest_neighbors_rbc(distance, n_neighbors, nrows):
     X, y = make_blobs(n_samples=nrows,
-                      n_features=2, random_state=0)
+                      centers=25,
+                      shuffle=True,
+                      n_features=2,
+                      cluster_std=3.0,
+                      random_state=42)
 
     knn_cu = cuKNN(metric=distance, algorithm="rbc")
     knn_cu.fit(X)
@@ -539,16 +543,14 @@ def test_nearest_neighbors_rbc(distance, n_neighbors, nrows):
         brute_d, brute_i = knn_cu_brute.kneighbors(
             X[:query_rows, :], n_neighbors=n_neighbors)
 
-    cp.testing.assert_allclose(rbc_d, brute_d, atol=5e-2,
-                               rtol=1e-3)
     rbc_i = cp.sort(rbc_i, axis=1)
     brute_i = cp.sort(brute_i, axis=1)
 
-    diff = rbc_i != brute_i
-
-    # Using a very small tolerance for subtle differences
-    # in indices that result from non-determinism
-    assert diff.ravel().sum() < 5
+    # TODO: These are failing with 1 or 2 mismatched elements
+    # for very small values of k:
+    # https://github.com/rapidsai/cuml/issues/4262
+    assert len(brute_d[brute_d != rbc_d]) <= 1
+    assert len(brute_i[brute_i != rbc_i]) <= 1
 
 
 @pytest.mark.parametrize("metric", valid_metrics_sparse())
@@ -599,13 +601,8 @@ def test_nearest_neighbors_sparse(metric,
     skD, skI = sknn.kneighbors(b.get())
 
     # For some reason, this will occasionally fail w/ a single
-    # mismatched element in CI. Try again if this happens.
-    try:
-        cp.testing.assert_allclose(cuD, skD, atol=1e-3, rtol=1e-3)
-    except AssertionError:
-        sknn.fit(sk_X)
-        skD, skI = sknn.kneighbors(b.get())
-        cp.testing.assert_allclose(cuD, skD, atol=1e-3, rtol=1e-3)
+    # mismatched element in CI. Allowing the single mismatch for now.
+    cp.testing.assert_allclose(cuD, skD, atol=1e-5, rtol=1e-5)
 
     # Jaccard & Chebyshev have a high potential for mismatched indices
     # due to duplicate distances. We can ignore the indices in this case.