From 9532b50adc8683ef25a8745f9def20aba9539e3b Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 2 Jun 2023 10:31:02 +0800 Subject: [PATCH 01/11] add random creation tests --- tests/integration/test_random_creation.py | 362 ++++++++++++++++++++++ 1 file changed, 362 insertions(+) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 4a6f09642..6e142ab19 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -13,6 +13,8 @@ # limitations under the License. # +from typing import Any, Tuple + import numpy as np import pytest from utils.random import assert_distribution @@ -33,6 +35,366 @@ def test_randn(): assert_distribution(a_num, 0.0, 1.0, mean_tol=0.05) +def reseed_and_gen_random( + func: str, seed: Any, *args: Any, **kwargs: Any +) -> Tuple[Any, Any]: + """Reseeed singleton rng and generate random in NumPy and cuNumeric.""" + np.random.seed(seed) + num.random.seed(seed) + return gen_random_from_both(func, *args, **kwargs) + + +def gen_random_from_both( + func: str, *args: Any, **kwargs: Any +) -> Tuple[Any, Any]: + """Call the same random function from both NumPy and cuNumeric.""" + return ( + getattr(np.random, func)(*args, **kwargs), + getattr(num.random, func)(*args, **kwargs), + ) + + +@pytest.mark.parametrize( + "seed", + [ + pytest.param( + 12345, + marks=pytest.mark.xfail( + reason="cuNumeric does not respect the singleton generator" + ), + # https://github.com/nv-legate/cunumeric/issues/601 + # NumPy: generates the same array after initializing with the seed. + # cuNumeric: keeps generating different arrays. + ), + None, + pytest.param( + (4, 6, 8), + marks=pytest.mark.xfail( + reason="cuNumeric does not take tuple as seed" + ), + # NumPy: pass + # cuNumeric: from runtime.set_next_random_epoch(int(init)): + # TypeError: int() argument must be a string, a bytes-like object + # or a real number, not 'tuple' + ), + ], + ids=str, +) +def test_singleton_seed(seed): + """Test that the singleton seed intitialize the sequence properly.""" + arr_np_1, arr_num_1 = reseed_and_gen_random("random", seed, 10000) + arr_np_2, arr_num_2 = reseed_and_gen_random("random", seed, 10000) + assert np.array_equal(arr_np_1, arr_np_2) == np.array_equal( + arr_num_1, arr_num_2 + ) + + +@pytest.mark.parametrize( + "seed", + [ + 12345, + pytest.param( + (0, 4, 5), + marks=pytest.mark.xfail( + reason="cuNumeric fails to generate random" + # NumPy: pass + # cuNumeric: struct.error: required argument is not an integer + ), + ), + ], + ids=str, +) +def test_default_rng_seed(seed): + rng_np_1, rng_num_1 = gen_random_from_both("default_rng", seed) + rng_np_2, rng_num_2 = gen_random_from_both("default_rng", seed) + assert rng_np_1.random() == rng_np_2.random() + assert rng_num_1.random() == rng_num_2.random() + + +def test_default_rng_bitgenerator(): + seed = 12345 + rng_np_1 = np.random.default_rng(np.random.PCG64(seed)) + rng_num_1 = num.random.default_rng(num.random.XORWOW(seed)) + rng_np_2 = np.random.default_rng(np.random.PCG64(seed)) + rng_num_2 = num.random.default_rng(num.random.XORWOW(seed)) + assert rng_np_1.random() == rng_np_2.random() + assert rng_num_1.random() == rng_num_2.random() + + +def test_default_rng_generator(): + steps = 3 + seed = 12345 + seed_rng_np, seed_rng_num = gen_random_from_both("default_rng", seed) + np_rngs = [np.random.default_rng(seed_rng_np) for _ in range(steps)] + num_rngs = [num.random.default_rng(seed_rng_num) for _ in range(steps)] + np_seq_1 = [rng.random() for rng in np_rngs] + num_seq_1 = [rng.random() for rng in num_rngs] + + rng_np, rng_num = gen_random_from_both("default_rng", seed) + np_seq_2 = [rng_np.random() for _ in range(steps)] + num_seq_2 = [rng_num.random() for _ in range(steps)] + assert np_seq_1 == np_seq_2 + assert num_seq_1 == num_seq_2 + + +@pytest.mark.parametrize("shape", [(20, 2, 31), (10000,)], ids=str) +def test_rand(shape): + arr_np = np.random.rand(*shape) + arr_num = np.random.rand(*shape) + assert arr_np.__class__.__name__ == arr_num.__class__.__name__ + if hasattr(arr_np, "__array__"): + assert arr_np.shape == arr_num.shape + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +LOW_HIGH = [ + (5, 10000), + (-10000, 5), + pytest.param( + 3000.45, + 15000, + marks=pytest.mark.xfail( + reason="NumPy pass, cuNumeric hit struct.error" + ), + # When size is greater than 1024, legate core throws error + # struct.error: required argument is not an integer + ), + (10000, None), +] +SIZES = [ + 10000, + pytest.param( + None, + marks=pytest.mark.xfail( + reason="NumPy returns scalar, cuNumeric returns 1-dim array" + # with dtype: + # np.random.randint(5, 10000, None, np.int64).ndim << 0 + # num.random.randint(5, 10000, None, np.int64).ndim << 1 + # without dtype: + # np.random.randint(5, 10000, None).__class__ + # + # num.random.randint(5, 10000, None).__class__ + # + ), + ), + 0, + (20, 50, 4), +] + + +@pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) +@pytest.mark.parametrize("size", SIZES, ids=str) +@pytest.mark.parametrize( + "dtype", + [ + np.int64, + pytest.param( + np.uint16, + marks=pytest.mark.xfail( + reason="cuNumeric raises NotImplementedError" + ), + ), + ], + ids=str, +) +def test_randint(low, high, size, dtype): + arr_np, arr_num = gen_random_from_both( + "randint", low=low, high=high, size=size, dtype=dtype + ) + assert arr_np.shape == arr_num.shape + # skip distribution assert for small arrays + if arr_np.size > 1024: + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +@pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") +@pytest.mark.parametrize("size", (1024, 1025)) +def test_randint_bool(size): + """Test randint with boolean output dtype.""" + arr_np, arr_num = gen_random_from_both("randint", 2, size=size, dtype=bool) + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + # NumPy pass + # cuNumeric LEGATE_TEST=1 or size > 1024: + # NotImplementedError: type for random.integers has to be int64 or int32 + # or int16 + + +@pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) +@pytest.mark.parametrize("size", SIZES, ids=str) +def test_random_integers(low, high, size): + arr_np, arr_num = gen_random_from_both( + "random_integers", low=low, high=high, size=size + ) + assert arr_np.shape == arr_num.shape + # skip distribution assert for small arrays + if arr_np.size > 1024: + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +@pytest.mark.parametrize("size", SIZES, ids=str) +def test_random_sample(size): + arr_np, arr_num = gen_random_from_both("random_sample", size=size) + assert arr_np.shape == arr_num.shape + # skip distribution assert for small arrays + if arr_np.size > 1024: + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +class TestRandomErrors: + def assert_exc_from_both(self, func, context, *args, **kwargs): + with context: + getattr(np.random, func)(*args, **kwargs) + with context: + getattr(num.random, func)(*args, **kwargs) + + @pytest.mark.parametrize( + "seed, expected_exc", + [ + (-100, pytest.raises(ValueError)), + (12.0, pytest.raises(TypeError)), + ("abc", pytest.raises(TypeError)), + ], + ids=lambda x: f" {str(getattr(x, 'expected_exception', x))} ", + ) + @pytest.mark.xfail(reason="NumPy raises exceptions, cuNumeric pass") + def test_invalid_seed(self, seed, expected_exc): + self.assert_exc_from_both("seed", expected_exc, seed) + # -100: NumPy raises ValueError: Seed must be between 0 and 2**32 - 1 + # 12.0: NumPy raises TypeError: Cannot cast scalar from + # dtype('float64') to dtype('int64') according to the rule 'safe' + # "abc": TypeError: Cannot cast scalar from dtype('= high + # cuNumeric LEGATE_TEST=1: array([-5410197118219848280]) + ), + ), + ], + ids=str, + ) + def test_randint_invalid_range(self, low, high, expected_exc): + self.assert_exc_from_both("randint", expected_exc, low, high) + + @pytest.mark.parametrize( + "size, expected_exc", + [ + (12.3, pytest.raises(TypeError)), + (-1, pytest.raises(ValueError)), + ((12, 4, 5.0), pytest.raises(TypeError)), + ], + ids=str, + ) + def test_randint_invalid_size(self, size, expected_exc): + self.assert_exc_from_both("randint", expected_exc, 10000, size=size) + + @pytest.mark.parametrize( + "dtype", + [ + pytest.param( + str, + marks=pytest.mark.xfail( + reason="NumPy raise TypeError, cuNumeric pass" + ), + ), + # NumPy: TypeError: Unsupported dtype dtype(' 1024 or LEGATE_TEST=1: + # NotImplementedError: type for random.integers has to be int64 or + # int32 or int16 + # cuNumeric size <= 1024 and LEGATE_TEST=0: returns array of booleans + + @pytest.mark.parametrize( + "size, expected_exc", + ( + (-1234, pytest.raises(ValueError)), + (32.5, pytest.raises(TypeError)), + ((9.6, 7), pytest.raises(TypeError)), + ("0", pytest.raises(TypeError)), + ), + ids=str, + ) + def test_random_sample_invalid_size(self, size, expected_exc): + self.assert_exc_from_both("random_sample", expected_exc, size=size) + + if __name__ == "__main__": import sys From d2bc433fd09303c89e03b78c034acba26e6f1a91 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Wed, 7 Jun 2023 12:36:34 +0800 Subject: [PATCH 02/11] Fix rand test case --- tests/integration/test_random_creation.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 6e142ab19..1e540d21b 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -139,14 +139,11 @@ def test_default_rng_generator(): @pytest.mark.parametrize("shape", [(20, 2, 31), (10000,)], ids=str) def test_rand(shape): - arr_np = np.random.rand(*shape) - arr_num = np.random.rand(*shape) - assert arr_np.__class__.__name__ == arr_num.__class__.__name__ - if hasattr(arr_np, "__array__"): - assert arr_np.shape == arr_num.shape - assert_distribution( - arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 - ) + arr_np, arr_num = gen_random_from_both("rand", *shape) + assert arr_np.shape == arr_num.shape + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) LOW_HIGH = [ From 15dccbe4207f7c56d47637047a92974d7a724af2 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Thu, 8 Jun 2023 09:55:25 +0800 Subject: [PATCH 03/11] Add int32 and int16 to test_randint --- tests/integration/test_random_creation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 1e540d21b..697b71221 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -187,6 +187,8 @@ def test_rand(shape): "dtype", [ np.int64, + np.int32, + np.int16, pytest.param( np.uint16, marks=pytest.mark.xfail( From 5c60ff28a9bc47251a59b17afc3d5c0122504c2f Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Thu, 8 Jun 2023 10:02:54 +0800 Subject: [PATCH 04/11] Move pytest.raises outside parameters --- tests/integration/test_random_creation.py | 44 +++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 697b71221..f8758b069 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -250,18 +250,18 @@ def test_random_sample(size): class TestRandomErrors: - def assert_exc_from_both(self, func, context, *args, **kwargs): - with context: + def assert_exc_from_both(self, func, exc, *args, **kwargs): + with pytest.raises(exc): getattr(np.random, func)(*args, **kwargs) - with context: + with pytest.raises(exc): getattr(num.random, func)(*args, **kwargs) @pytest.mark.parametrize( "seed, expected_exc", [ - (-100, pytest.raises(ValueError)), - (12.0, pytest.raises(TypeError)), - ("abc", pytest.raises(TypeError)), + (-100, ValueError), + (12.0, TypeError), + ("abc", TypeError), ], ids=lambda x: f" {str(getattr(x, 'expected_exception', x))} ", ) @@ -279,11 +279,11 @@ def test_invalid_seed(self, seed, expected_exc): @pytest.mark.parametrize( "shape, expected_exc", [ - (-100, pytest.raises(ValueError)), - (12.0, pytest.raises(TypeError)), - ("abc", pytest.raises(TypeError)), - (None, pytest.raises(TypeError)), - ((12345,), pytest.raises(TypeError)), + (-100, ValueError), + (12.0, TypeError), + ("abc", TypeError), + (None, TypeError), + ((12345,), TypeError), ], ids=lambda x: f" {str(getattr(x, 'expected_exception', x))} ", ) @@ -296,7 +296,7 @@ def test_rand_invalid_shape(self, shape, expected_exc): pytest.param( -10000, None, - pytest.raises(ValueError), + ValueError, marks=pytest.mark.xfail( reason="cuNumeric does not raise error when LEGATE_TEST=1" # NumPy & cuNumeric LEGATE_TEST=0: ValueError: high <= 0 @@ -306,7 +306,7 @@ def test_rand_invalid_shape(self, shape, expected_exc): pytest.param( -10000, -20000, - pytest.raises(ValueError), + ValueError, marks=pytest.mark.xfail( reason="cuNumeric does not raise error when LEGATE_TEST=1" # NumPy & cuNumeric LEGATE_TEST=0: ValueError: low >= high @@ -322,9 +322,9 @@ def test_randint_invalid_range(self, low, high, expected_exc): @pytest.mark.parametrize( "size, expected_exc", [ - (12.3, pytest.raises(TypeError)), - (-1, pytest.raises(ValueError)), - ((12, 4, 5.0), pytest.raises(TypeError)), + (12.3, TypeError), + (-1, ValueError), + ((12, 4, 5.0), TypeError), ], ids=str, ) @@ -364,13 +364,13 @@ def test_randint_invalid_size(self, size, expected_exc): ids=str, ) def test_randint_dtype(self, dtype): - expected_exc = pytest.raises(TypeError) + expected_exc = TypeError self.assert_exc_from_both("randint", expected_exc, 10000, dtype=dtype) @pytest.mark.xfail(reason="cuNumeric pass or raise NotImplementedError") @pytest.mark.parametrize("size", (1024, 1025)) def test_randint_bool(self, size): - expected_exc = pytest.raises(ValueError) + expected_exc = ValueError self.assert_exc_from_both( "randint", expected_exc, 10000, size=size, dtype=bool ) @@ -383,10 +383,10 @@ def test_randint_bool(self, size): @pytest.mark.parametrize( "size, expected_exc", ( - (-1234, pytest.raises(ValueError)), - (32.5, pytest.raises(TypeError)), - ((9.6, 7), pytest.raises(TypeError)), - ("0", pytest.raises(TypeError)), + (-1234, ValueError), + (32.5, TypeError), + ((9.6, 7), TypeError), + ("0", TypeError), ), ids=str, ) From edab0d24bd5947afdbd59db764ef98fc7fb81928 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 9 Jun 2023 12:58:59 +0800 Subject: [PATCH 05/11] Reorganize the test cases --- tests/integration/test_random_creation.py | 175 +++++++++++++++------- 1 file changed, 120 insertions(+), 55 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index f8758b069..e8a813910 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -149,65 +149,112 @@ def test_rand(shape): LOW_HIGH = [ (5, 10000), (-10000, 5), - pytest.param( - 3000.45, - 15000, - marks=pytest.mark.xfail( - reason="NumPy pass, cuNumeric hit struct.error" - ), - # When size is greater than 1024, legate core throws error - # struct.error: required argument is not an integer - ), - (10000, None), -] -SIZES = [ - 10000, - pytest.param( - None, - marks=pytest.mark.xfail( - reason="NumPy returns scalar, cuNumeric returns 1-dim array" - # with dtype: - # np.random.randint(5, 10000, None, np.int64).ndim << 0 - # num.random.randint(5, 10000, None, np.int64).ndim << 1 - # without dtype: - # np.random.randint(5, 10000, None).__class__ - # - # num.random.randint(5, 10000, None).__class__ - # - ), - ), - 0, - (20, 50, 4), ] +SMALL_RNG_SIZES = [5, 1024, (1, 2)] +LARGE_RNG_SIZES = [10000, (20, 50, 4)] +ALL_RNG_SIZES = SMALL_RNG_SIZES + LARGE_RNG_SIZES +INT_DTYPES = [np.int64, np.int32, np.int16] +UINT_DTYPES = [np.uint64, np.uint16, np.uint0] +@pytest.mark.parametrize("size", ALL_RNG_SIZES, ids=str) @pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) -@pytest.mark.parametrize("size", SIZES, ids=str) +@pytest.mark.parametrize("dtype", INT_DTYPES, ids=str) +def test_randint_basic_stats(low, high, size, dtype): + arr_np, arr_num = gen_random_from_both( + "randint", low=low, high=high, size=size, dtype=dtype + ) + assert arr_np.dtype == arr_np.dtype + assert arr_np.shape == arr_num.shape + assert np.min(arr_num) >= low + assert np.max(arr_num) <= high + + @pytest.mark.parametrize( - "dtype", + "low", [ - np.int64, - np.int32, - np.int16, pytest.param( - np.uint16, + 0, marks=pytest.mark.xfail( - reason="cuNumeric raises NotImplementedError" + reason="cuNumeric allows 0 as higher bound." + # NumPy: ValueError: high <= 0 + # cuNumeric: array([0]) ), ), + 1024, + 1025, ], ids=str, ) -def test_randint(low, high, size, dtype): +@pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) +@pytest.mark.parametrize("dtype", INT_DTYPES, ids=str) +def test_randint_low_range_only(low, size, dtype): + arr_np, arr_num = gen_random_from_both( + "randint", low, size=size, dtype=dtype + ) + assert arr_np.dtype == arr_np.dtype + assert arr_np.shape == arr_num.shape + assert np.max(arr_num) < low + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +@pytest.mark.xfail +@pytest.mark.parametrize( + "low, high", [(3000.45, 15000), (123, 45.6), (12.3, 45.6)], ids=str +) +def test_randint_float_range(low, high): + # When size is greater than 1024, legate core throws error + # struct.error: required argument is not an integer + arr_np, arr_num = gen_random_from_both( + "randint", low=low, high=high, size=1025 + ) + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) + + +@pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") +@pytest.mark.parametrize("size", ALL_RNG_SIZES, ids=str) +@pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) +@pytest.mark.parametrize("dtype", UINT_DTYPES, ids=str) +def test_randint_uint(low, high, dtype, size): + # NotImplementedError: type for random.integers has to be int64 or int32 + # or int16 arr_np, arr_num = gen_random_from_both( "randint", low=low, high=high, size=size, dtype=dtype ) + assert arr_np.dtype == arr_np.dtype assert arr_np.shape == arr_num.shape - # skip distribution assert for small arrays - if arr_np.size > 1024: - assert_distribution( - arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 - ) + assert np.min(arr_num) >= low + assert np.max(arr_num) <= high + + +@pytest.mark.xfail( + reason="NumPy returns scalar, cuNumeric returns 1-dim array" +) +def test_randint_size_none(): + arr_np, arr_num = gen_random_from_both("randint", 1234, size=None) + assert np.isscalar(arr_np) == np.isscalar(arr_num) + + +def test_randint_size_zero(): + arr_np, arr_num = gen_random_from_both("randint", 1234, size=0) + assert arr_np.dtype == arr_np.dtype + assert arr_np.shape == arr_np.shape + + +@pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) +@pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) +@pytest.mark.parametrize("dtype", INT_DTYPES) +def test_randint_distribution(low, high, size, dtype): + arr_np, arr_num = gen_random_from_both( + "randint", low=low, high=high, size=size, dtype=dtype + ) + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) @pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") @@ -225,28 +272,37 @@ def test_randint_bool(size): @pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) -@pytest.mark.parametrize("size", SIZES, ids=str) +@pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) def test_random_integers(low, high, size): arr_np, arr_num = gen_random_from_both( "random_integers", low=low, high=high, size=size ) assert arr_np.shape == arr_num.shape - # skip distribution assert for small arrays - if arr_np.size > 1024: - assert_distribution( - arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 - ) + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) -@pytest.mark.parametrize("size", SIZES, ids=str) -def test_random_sample(size): +@pytest.mark.parametrize("size", ALL_RNG_SIZES, ids=str) +def test_random_sample_basic_stats(size): arr_np, arr_num = gen_random_from_both("random_sample", size=size) assert arr_np.shape == arr_num.shape - # skip distribution assert for small arrays - if arr_np.size > 1024: - assert_distribution( - arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 - ) + + +@pytest.mark.xfail( + reason="NumPy returns scalar, cuNumeric returns 1-dim array" +) +def test_random_sample_size_none(): + arr_np, arr_num = gen_random_from_both("random_sample", size=None) + assert np.isscalar(arr_np) == num.isscalar(arr_num) + + +@pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) +def test_random_sample(size): + arr_np, arr_num = gen_random_from_both("random_sample", size=size) + assert_distribution( + arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 + ) class TestRandomErrors: @@ -331,6 +387,15 @@ def test_randint_invalid_range(self, low, high, expected_exc): def test_randint_invalid_size(self, size, expected_exc): self.assert_exc_from_both("randint", expected_exc, 10000, size=size) + @pytest.mark.xfail(reason="cuNumeric does not check the bound") + def test_randint_int16_bound(self, high, dtype): + # NumPy: ValueError: high is out of bounds for int16 + # cuNumeric: array([13642], dtype=int16) + expected_exc = ValueError + self.assert_exc_from_both( + "randint", expected_exc, 34567, dtype=np.int16 + ) + @pytest.mark.parametrize( "dtype", [ From ac6d95f6cd3171f9627e06b6353d1b10f44177fa Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 9 Jun 2023 13:06:22 +0800 Subject: [PATCH 06/11] move 0 higher bound test to the error test class. --- tests/integration/test_random_creation.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index e8a813910..3952a9fe1 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -170,22 +170,7 @@ def test_randint_basic_stats(low, high, size, dtype): assert np.max(arr_num) <= high -@pytest.mark.parametrize( - "low", - [ - pytest.param( - 0, - marks=pytest.mark.xfail( - reason="cuNumeric allows 0 as higher bound." - # NumPy: ValueError: high <= 0 - # cuNumeric: array([0]) - ), - ), - 1024, - 1025, - ], - ids=str, -) +@pytest.mark.parametrize("low", [1024, 1025, 12345], ids=str) @pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) @pytest.mark.parametrize("dtype", INT_DTYPES, ids=str) def test_randint_low_range_only(low, size, dtype): @@ -396,6 +381,11 @@ def test_randint_int16_bound(self, high, dtype): "randint", expected_exc, 34567, dtype=np.int16 ) + @pytest.mark.xfail(reason="cuNumeric does not check the bound") + def test_randint_higher_bound_zero(self): + expected_exc = ValueError + self.assert_exc_from_both("randint", expected_exc, 0) + @pytest.mark.parametrize( "dtype", [ From 4d069f4a17dc34b62428dadcc0894370b4b3d476 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Fri, 9 Jun 2023 16:53:05 +0800 Subject: [PATCH 07/11] fine tune xfail conditions for eager mode. --- tests/integration/test_random_creation.py | 65 +++++++++++++++++------ 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 3952a9fe1..d2a524998 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -13,6 +13,7 @@ # limitations under the License. # +import os from typing import Any, Tuple import numpy as np @@ -21,6 +22,9 @@ import cunumeric as num +LEGATE_TEST = os.environ.get("LEGATE_TEST", None) == "1" +EAGER_TEST = os.environ.get("CUNUMERIC_FORCE_THUNK", None) == "eager" + def test_randn(): # We cannot expect that random generation will match with NumPy's, @@ -60,11 +64,13 @@ def gen_random_from_both( pytest.param( 12345, marks=pytest.mark.xfail( - reason="cuNumeric does not respect the singleton generator" + not EAGER_TEST, + reason="cuNumeric does not respect the singleton generator", ), # https://github.com/nv-legate/cunumeric/issues/601 # NumPy: generates the same array after initializing with the seed. # cuNumeric: keeps generating different arrays. + # seed is respected in Eager mode. ), None, pytest.param( @@ -89,6 +95,10 @@ def test_singleton_seed(seed): ) +@pytest.mark.xfail( + EAGER_TEST, + reason="cuNumeric does not respect seed in Eager mode", +) @pytest.mark.parametrize( "seed", [ @@ -111,6 +121,10 @@ def test_default_rng_seed(seed): assert rng_num_1.random() == rng_num_2.random() +@pytest.mark.xfail( + EAGER_TEST, + reason="cuNumeric does not respect seed in Eager mode", +) def test_default_rng_bitgenerator(): seed = 12345 rng_np_1 = np.random.default_rng(np.random.PCG64(seed)) @@ -121,6 +135,10 @@ def test_default_rng_bitgenerator(): assert rng_num_1.random() == rng_num_2.random() +@pytest.mark.xfail( + EAGER_TEST, + reason="cuNumeric does not respect seed in Eager mode", +) def test_default_rng_generator(): steps = 3 seed = 12345 @@ -173,21 +191,31 @@ def test_randint_basic_stats(low, high, size, dtype): @pytest.mark.parametrize("low", [1024, 1025, 12345], ids=str) @pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str) @pytest.mark.parametrize("dtype", INT_DTYPES, ids=str) -def test_randint_low_range_only(low, size, dtype): +def test_randint_low_limit_only(low, size, dtype): arr_np, arr_num = gen_random_from_both( "randint", low, size=size, dtype=dtype ) assert arr_np.dtype == arr_np.dtype assert arr_np.shape == arr_num.shape - assert np.max(arr_num) < low assert_distribution( arr_num, np.mean(arr_np), np.std(arr_np), mean_tol=0.05 ) -@pytest.mark.xfail +@pytest.mark.xfail( + EAGER_TEST, + reason="cuNumeric randint high limit is inclusive in Eager mode", +) +def test_randint_high_limit(): + limit = 10 + arr_np, arr_num = gen_random_from_both("randint", 10, size=100) + assert np.max(arr_np) < limit + assert np.max(arr_num) < limit + + +@pytest.mark.xfail(not EAGER_TEST, reason="cuNumeric hits struct error.") @pytest.mark.parametrize( - "low, high", [(3000.45, 15000), (123, 45.6), (12.3, 45.6)], ids=str + "low, high", [(3000.45, 15000), (123, 456.7), (12.3, 45.6)], ids=str ) def test_randint_float_range(low, high): # When size is greater than 1024, legate core throws error @@ -200,9 +228,11 @@ def test_randint_float_range(low, high): ) -@pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") +@pytest.mark.xfail( + not EAGER_TEST, reason="cuNumeric raises NotImplementedError" +) @pytest.mark.parametrize("size", ALL_RNG_SIZES, ids=str) -@pytest.mark.parametrize("low, high", LOW_HIGH, ids=str) +@pytest.mark.parametrize("low, high", [(1000, 65535), (0, 1024)], ids=str) @pytest.mark.parametrize("dtype", UINT_DTYPES, ids=str) def test_randint_uint(low, high, dtype, size): # NotImplementedError: type for random.integers has to be int64 or int32 @@ -331,6 +361,10 @@ def test_invalid_seed(self, seed, expected_exc): def test_rand_invalid_shape(self, shape, expected_exc): self.assert_exc_from_both("rand", expected_exc, shape) + @pytest.mark.xfail( + LEGATE_TEST and not EAGER_TEST, + reason="cuNumeric does not raise error when LEGATE_TEST=1", + ) @pytest.mark.parametrize( "low, high, expected_exc", [ @@ -338,26 +372,23 @@ def test_rand_invalid_shape(self, shape, expected_exc): -10000, None, ValueError, - marks=pytest.mark.xfail( - reason="cuNumeric does not raise error when LEGATE_TEST=1" - # NumPy & cuNumeric LEGATE_TEST=0: ValueError: high <= 0 - # cuNumeric LEGATE_TEST=1: array([3124366888496675138]) - ), ), pytest.param( -10000, -20000, ValueError, - marks=pytest.mark.xfail( - reason="cuNumeric does not raise error when LEGATE_TEST=1" - # NumPy & cuNumeric LEGATE_TEST=0: ValueError: low >= high - # cuNumeric LEGATE_TEST=1: array([-5410197118219848280]) - ), ), ], ids=str, ) def test_randint_invalid_range(self, low, high, expected_exc): + # When LEGATE_TEST=1: + # (-10000, None): + # NumPy & cuNumeric LEGATE_TEST=0: ValueError: high <= 0 + # cuNumeric LEGATE_TEST=1: array([3124366888496675138]) + # (-10000, -20000): + # NumPy & cuNumeric LEGATE_TEST=0: ValueError: low >= high + # cuNumeric LEGATE_TEST=1: array([-5410197118219848280]) self.assert_exc_from_both("randint", expected_exc, low, high) @pytest.mark.parametrize( From 0758ad0b770838a5cccb0ea6d42c627f7e6b2a03 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Mon, 12 Jun 2023 11:07:06 +0800 Subject: [PATCH 08/11] Conditionally skip test_randint_higher_bound_zero. --- tests/integration/test_random_creation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index d2a524998..4cedbeb8f 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -404,7 +404,7 @@ def test_randint_invalid_size(self, size, expected_exc): self.assert_exc_from_both("randint", expected_exc, 10000, size=size) @pytest.mark.xfail(reason="cuNumeric does not check the bound") - def test_randint_int16_bound(self, high, dtype): + def test_randint_int16_bound(self): # NumPy: ValueError: high is out of bounds for int16 # cuNumeric: array([13642], dtype=int16) expected_exc = ValueError @@ -412,6 +412,11 @@ def test_randint_int16_bound(self, high, dtype): "randint", expected_exc, 34567, dtype=np.int16 ) + @pytest.mark.xfail( + os.environ.get("REALM_SYNTHETIC_CORE_MAP", None) == "", + reason="cunumeric hit FPE", + run=False, + ) @pytest.mark.xfail(reason="cuNumeric does not check the bound") def test_randint_higher_bound_zero(self): expected_exc = ValueError From d196e0427d1b7682b26fc7f8880120ca7602dd8e Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Mon, 12 Jun 2023 14:36:10 +0800 Subject: [PATCH 09/11] Use skip instead of xfail --- tests/integration/test_random_creation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 4cedbeb8f..46a449f3d 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -412,10 +412,9 @@ def test_randint_int16_bound(self): "randint", expected_exc, 34567, dtype=np.int16 ) - @pytest.mark.xfail( + @pytest.mark.skipif( os.environ.get("REALM_SYNTHETIC_CORE_MAP", None) == "", reason="cunumeric hit FPE", - run=False, ) @pytest.mark.xfail(reason="cuNumeric does not check the bound") def test_randint_higher_bound_zero(self): From 92c08f2a3daa4c6dca2ffafcf2b680f0701a27a8 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Wed, 14 Jun 2023 15:27:18 +0800 Subject: [PATCH 10/11] update/remove xfail according to the latest behavior --- tests/integration/test_random_creation.py | 32 ++++++----------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 46a449f3d..1770c5e90 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -202,10 +202,6 @@ def test_randint_low_limit_only(low, size, dtype): ) -@pytest.mark.xfail( - EAGER_TEST, - reason="cuNumeric randint high limit is inclusive in Eager mode", -) def test_randint_high_limit(): limit = 10 arr_np, arr_num = gen_random_from_both("randint", 10, size=100) @@ -213,13 +209,15 @@ def test_randint_high_limit(): assert np.max(arr_num) < limit -@pytest.mark.xfail(not EAGER_TEST, reason="cuNumeric hits struct error.") +@pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") @pytest.mark.parametrize( "low, high", [(3000.45, 15000), (123, 456.7), (12.3, 45.6)], ids=str ) def test_randint_float_range(low, high): - # When size is greater than 1024, legate core throws error - # struct.error: required argument is not an integer + # NumPy returns integer scalar + # cuNumeric raises one of the following + # NotImplementedError: 'low' must be an integer + # NotImplementedError: 'high' must be an integer or None arr_np, arr_num = gen_random_from_both( "randint", low=low, high=high, size=1025 ) @@ -272,7 +270,9 @@ def test_randint_distribution(low, high, size, dtype): ) -@pytest.mark.xfail(reason="cuNumeric raises NotImplementedError") +@pytest.mark.xfail( + not EAGER_TEST, reason="cuNumeric raises NotImplementedError" +) @pytest.mark.parametrize("size", (1024, 1025)) def test_randint_bool(size): """Test randint with boolean output dtype.""" @@ -361,10 +361,6 @@ def test_invalid_seed(self, seed, expected_exc): def test_rand_invalid_shape(self, shape, expected_exc): self.assert_exc_from_both("rand", expected_exc, shape) - @pytest.mark.xfail( - LEGATE_TEST and not EAGER_TEST, - reason="cuNumeric does not raise error when LEGATE_TEST=1", - ) @pytest.mark.parametrize( "low, high, expected_exc", [ @@ -382,13 +378,6 @@ def test_rand_invalid_shape(self, shape, expected_exc): ids=str, ) def test_randint_invalid_range(self, low, high, expected_exc): - # When LEGATE_TEST=1: - # (-10000, None): - # NumPy & cuNumeric LEGATE_TEST=0: ValueError: high <= 0 - # cuNumeric LEGATE_TEST=1: array([3124366888496675138]) - # (-10000, -20000): - # NumPy & cuNumeric LEGATE_TEST=0: ValueError: low >= high - # cuNumeric LEGATE_TEST=1: array([-5410197118219848280]) self.assert_exc_from_both("randint", expected_exc, low, high) @pytest.mark.parametrize( @@ -412,11 +401,6 @@ def test_randint_int16_bound(self): "randint", expected_exc, 34567, dtype=np.int16 ) - @pytest.mark.skipif( - os.environ.get("REALM_SYNTHETIC_CORE_MAP", None) == "", - reason="cunumeric hit FPE", - ) - @pytest.mark.xfail(reason="cuNumeric does not check the bound") def test_randint_higher_bound_zero(self): expected_exc = ValueError self.assert_exc_from_both("randint", expected_exc, 0) From e29302415f44fb3f6c09fe0968a17b5ed934bd82 Mon Sep 17 00:00:00 2001 From: Yimo Jiang Date: Mon, 26 Jun 2023 13:42:57 +0800 Subject: [PATCH 11/11] remove xfail from test_randint_size_none --- tests/integration/test_random_creation.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_random_creation.py b/tests/integration/test_random_creation.py index 1770c5e90..2acd1df6b 100644 --- a/tests/integration/test_random_creation.py +++ b/tests/integration/test_random_creation.py @@ -244,12 +244,9 @@ def test_randint_uint(low, high, dtype, size): assert np.max(arr_num) <= high -@pytest.mark.xfail( - reason="NumPy returns scalar, cuNumeric returns 1-dim array" -) def test_randint_size_none(): arr_np, arr_num = gen_random_from_both("randint", 1234, size=None) - assert np.isscalar(arr_np) == np.isscalar(arr_num) + assert np.ndim(arr_np) == np.ndim(arr_num) def test_randint_size_zero(): @@ -309,7 +306,7 @@ def test_random_sample_basic_stats(size): ) def test_random_sample_size_none(): arr_np, arr_num = gen_random_from_both("random_sample", size=None) - assert np.isscalar(arr_np) == num.isscalar(arr_num) + assert np.ndim(arr_np) == np.ndim(arr_num) @pytest.mark.parametrize("size", LARGE_RNG_SIZES, ids=str)