diff --git a/python/tvm/testing.py b/python/tvm/testing.py index 79518ac24984..6eeb9ab03f60 100644 --- a/python/tvm/testing.py +++ b/python/tvm/testing.py @@ -375,11 +375,12 @@ def _check_forward(constraints1, constraints2, varmap, backvarmap): def _get_targets(target_str=None): if target_str is None: target_str = os.environ.get("TVM_TEST_TARGETS", "") + # Use dict instead of set for de-duplication so that the + # targets stay in the order specified. + target_names = list({t.strip(): None for t in target_str.split(";") if t.strip()}) - if len(target_str) == 0: - target_str = DEFAULT_TEST_TARGETS - - target_names = set(t.strip() for t in target_str.split(";") if t.strip()) + if not target_names: + target_names = DEFAULT_TEST_TARGETS targets = [] for target in target_names: @@ -413,10 +414,18 @@ def _get_targets(target_str=None): return targets -DEFAULT_TEST_TARGETS = ( - "llvm;cuda;opencl;metal;rocm;vulkan -from_device=0;nvptx;" - "llvm -device=arm_cpu;opencl -device=mali,aocl_sw_emu" -) +DEFAULT_TEST_TARGETS = [ + "llvm", + "llvm -device=arm_cpu", + "cuda", + "nvptx", + "vulkan -from_device=0", + "opencl", + "opencl -device=mali,aocl_sw_emu", + "opencl -device=intel_graphics", + "metal", + "rocm", +] def device_enabled(target): @@ -730,20 +739,25 @@ def requires_rpc(*args): def _target_to_requirement(target): + if isinstance(target, str): + target_kind = target.split()[0] + else: + target_kind = target.kind.name + # mapping from target to decorator - if target.startswith("cuda"): + if target_kind == "cuda": return requires_cuda() - if target.startswith("rocm"): + if target_kind == "rocm": return requires_rocm() - if target.startswith("vulkan"): + if target_kind == "vulkan": return requires_vulkan() - if target.startswith("nvptx"): + if target_kind == "nvptx": return requires_nvptx() - if target.startswith("metal"): + if target_kind == "metal": return requires_metal() - if target.startswith("opencl"): + if target_kind == "opencl": return requires_opencl() - if target.startswith("llvm"): + if target_kind == "llvm": return requires_llvm() return [] @@ -794,16 +808,74 @@ def _auto_parametrize_target(metafunc): file. """ + + def update_parametrize_target_arg( + argnames, + argvalues, + *args, + **kwargs, + ): + args = [arg.strip() for arg in argnames.split(",") if arg.strip()] + if "target" in args: + target_i = args.index("target") + + new_argvalues = [] + for argvalue in argvalues: + + if isinstance(argvalue, _pytest.mark.structures.ParameterSet): + # The parametrized value is already a + # pytest.param, so track any marks already + # defined. + param_set = argvalue.values + target = param_set[target_i] + additional_marks = argvalue.marks + elif len(args) == 1: + # Single value parametrization, argvalue is a list of values. + target = argvalue + param_set = (target,) + additional_marks = [] + else: + # Multiple correlated parameters, argvalue is a list of tuple of values. + param_set = argvalue + target = param_set[target_i] + additional_marks = [] + + new_argvalues.append( + pytest.param( + *param_set, marks=_target_to_requirement(target) + additional_marks + ) + ) + + try: + argvalues[:] = new_argvalues + except TypeError as e: + pyfunc = metafunc.definition.function + filename = pyfunc.__code__.co_filename + line_number = pyfunc.__code__.co_firstlineno + msg = ( + f"Unit test {metafunc.function.__name__} ({filename}:{line_number}) " + "is parametrized using a tuple of parameters instead of a list " + "of parameters." + ) + raise TypeError(msg) from e + if "target" in metafunc.fixturenames: + # Update any explicit use of @pytest.mark.parmaetrize to + # parametrize over targets. This adds the appropriate + # @tvm.testing.requires_* markers for each target. + for mark in metafunc.definition.iter_markers("parametrize"): + update_parametrize_target_arg(*mark.args, **mark.kwargs) + + # Check if any explicit parametrizations exist, and apply one + # if they do not. If the function is marked with either + # excluded or known failing targets, use these to determine + # the targets to be used. parametrized_args = [ arg.strip() for mark in metafunc.definition.iter_markers("parametrize") for arg in mark.args[0].split(",") ] - if "target" not in parametrized_args: - # Check if the function is marked with either excluded or - # known failing targets. excluded_targets = getattr(metafunc.function, "tvm_excluded_targets", []) xfail_targets = getattr(metafunc.function, "tvm_known_failing_targets", []) metafunc.parametrize( @@ -849,17 +921,14 @@ def parametrize_targets(*args): >>> ... # do something """ - def wrap(targets): - def func(f): - return pytest.mark.parametrize( - "target", _pytest_target_params(targets), scope="session" - )(f) - - return func - + # Backwards compatibility, when used as a decorator with no + # arguments implicitly parametrizes over "target". The + # parametrization is now handled by _auto_parametrize_target, so + # this use case can just return the decorated function. if len(args) == 1 and callable(args[0]): - return wrap(None)(args[0]) - return wrap(args) + return args[0] + + return pytest.mark.parametrize("target", list(args), scope="session") def exclude_targets(*args): diff --git a/tests/python/frontend/onnx/test_forward.py b/tests/python/frontend/onnx/test_forward.py index 6a5ffd3821a1..06832a9cbf62 100644 --- a/tests/python/frontend/onnx/test_forward.py +++ b/tests/python/frontend/onnx/test_forward.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import glob import os import re import glob @@ -1604,11 +1605,9 @@ def verify_hardsigmoid(input_dim, alpha, beta): # TODO (mbrookhart, electriclilies) Fix argmin on GPU and enable this test +@tvm.testing.known_failing_targets("cuda") @tvm.testing.parametrize_targets def test_forward_arg_min_max(target, dev): - if "cuda" in target: - pytest.skip("Fails on CUDA") - def verify_argreduce(input_dim, op_name, axis=None, keepdims=None): a_np1 = np.random.uniform(-10, 10, input_dim).astype(np.int32) out_shape = list(a_np1.shape) @@ -4635,318 +4634,326 @@ def verify_eyelike(indata): are not yet supported are skipped. """ -f = onnx.__file__ +onnx_test_node_dir = os.path.join(os.path.dirname(onnx.__file__), "backend", "test", "data", "node") -onnx_test_folders = sorted(glob.glob("/".join(f.split("/")[0:-1]) + "/backend/test/data/node/*/")) +onnx_test_folders = sorted( + dirname + for dirname in os.listdir(onnx_test_node_dir) + if dirname.startswith("test") and os.path.isdir(os.path.join(onnx_test_node_dir, dirname)) +) unsupported_onnx_tests = [ - "test_adagrad/", - "test_adagrad_multiple/", - "test_adam/", - "test_adam_multiple/", - "test_argmax_default_axis_example_select_last_index/", - "test_argmax_default_axis_random_select_last_index/", - "test_argmax_keepdims_example_select_last_index/", - "test_argmax_keepdims_random_select_last_index/", - "test_argmax_negative_axis_keepdims_example_select_last_index/", - "test_argmax_negative_axis_keepdims_random_select_last_index/", - "test_argmax_no_keepdims_example_select_last_index/", - "test_argmax_no_keepdims_random_select_last_index/", - "test_argmin_default_axis_example_select_last_index/", - "test_argmin_default_axis_random_select_last_index/", - "test_argmin_keepdims_example_select_last_index/", - "test_argmin_keepdims_random_select_last_index/", - "test_argmin_negative_axis_keepdims_example_select_last_index/", - "test_argmin_negative_axis_keepdims_random_select_last_index/", - "test_argmin_no_keepdims_example_select_last_index/", - "test_argmin_no_keepdims_random_select_last_index/", - "test_cast_BFLOAT16_to_FLOAT/", - "test_cast_DOUBLE_to_FLOAT16/", - "test_cast_FLOAT_to_BFLOAT16/", - "test_cast_FLOAT_to_STRING/", - "test_cast_STRING_to_FLOAT/", - "test_celu/", - "test_compress_0/", - "test_compress_1/", - "test_compress_default_axis/", - "test_compress_negative_axis/", - "test_convtranspose_dilations/", - "test_convtranspose_output_shape/", - "test_cumsum_1d/", - "test_cumsum_1d_exclusive/", - "test_cumsum_1d_reverse/", - "test_cumsum_1d_reverse_exclusive/", - "test_cumsum_2d_axis_0/", - "test_cumsum_2d_axis_1/", - "test_cumsum_2d_negative_axis/", - "test_det_2d/", - "test_det_nd/", - "test_dropout_default/", - "test_dropout_default_mask/", - "test_dropout_default_mask_ratio/", - "test_dropout_default_ratio/", - "test_einsum_batch_diagonal/", - "test_einsum_batch_matmul/", - "test_einsum_inner_prod/", - "test_einsum_sum/", - "test_einsum_transpose/", - "test_greater_equal/", - "test_greater_equal_bcast/", - "test_hardmax_axis_0/", - "test_hardmax_axis_1/", - "test_hardmax_default_axis/", - "test_if_seq/", - "test_less_equal/", - "test_less_equal_bcast/", - "test_logsoftmax_axis_0/", - "test_logsoftmax_axis_0_expanded/", - "test_logsoftmax_axis_1/", - "test_logsoftmax_axis_1_expanded/", - "test_logsoftmax_axis_2_expanded/", - "test_logsoftmax_default_axis/", - "test_logsoftmax_default_axis_expanded/", - "test_logsoftmax_example_1_expanded/", - "test_logsoftmax_large_number_expanded/", - "test_logsoftmax_negative_axis_expanded/", - "test_loop11/", - "test_loop13_seq/", - "test_matmulinteger/", - "test_maxpool_2d_same_lower/", - "test_maxpool_2d_same_upper/", - "test_maxpool_with_argmax_2d_precomputed_pads/", - "test_maxpool_with_argmax_2d_precomputed_strides/", - "test_maxunpool_export_with_output_shape/", - "test_momentum/", - "test_momentum_multiple/", - "test_mvn/", - "test_nesterov_momentum/", - "test_nllloss_NC/", - "test_nllloss_NC_expanded/", - "test_nllloss_NCd1/", - "test_nllloss_NCd1_expanded/", - "test_nllloss_NCd1_ii/", - "test_nllloss_NCd1_ii_expanded/", - "test_nllloss_NCd1_mean_weight_negative_ii/", - "test_nllloss_NCd1_mean_weight_negative_ii_expanded/", - "test_nllloss_NCd1_weight/", - "test_nllloss_NCd1_weight_expanded/", - "test_nllloss_NCd1_weight_ii/", - "test_nllloss_NCd1_weight_ii_expanded/", - "test_nllloss_NCd1d2/", - "test_nllloss_NCd1d2_expanded/", - "test_nllloss_NCd1d2_no_weight_reduction_mean_ii/", - "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded/", - "test_nllloss_NCd1d2_reduction_mean/", - "test_nllloss_NCd1d2_reduction_mean_expanded/", - "test_nllloss_NCd1d2_reduction_sum/", - "test_nllloss_NCd1d2_reduction_sum_expanded/", - "test_nllloss_NCd1d2_with_weight/", - "test_nllloss_NCd1d2_with_weight_expanded/", - "test_nllloss_NCd1d2_with_weight_reduction_mean/", - "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded/", - "test_nllloss_NCd1d2_with_weight_reduction_sum/", - "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded/", - "test_nllloss_NCd1d2_with_weight_reduction_sum_ii/", - "test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded/", - "test_nllloss_NCd1d2d3_none_no_weight_negative_ii/", - "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded/", - "test_nllloss_NCd1d2d3_sum_weight_high_ii/", - "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded/", - "test_nllloss_NCd1d2d3d4d5_mean_weight/", - "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded/", - "test_nllloss_NCd1d2d3d4d5_none_no_weight/", - "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded/", - "test_pow_types_float/", - "test_pow_types_float32_int32/", - "test_pow_types_float32_int64/", - "test_pow_types_float32_uint32/", - "test_pow_types_float32_uint64/", - "test_pow_types_int/", - "test_pow_types_int32_float32/", - "test_pow_types_int32_int32/", - "test_pow_types_int64_float32/", - "test_pow_types_int64_int64/", - "test_qlinearmatmul_2D/", - "test_qlinearmatmul_3D/", - "test_range_float_type_positive_delta_expanded/", - "test_range_int32_type_negative_delta_expanded/", - "test_reduce_sum_default_axes_keepdims_example/", - "test_reduce_sum_default_axes_keepdims_random/", - "test_reduce_sum_do_not_keepdims_example/", - "test_reduce_sum_do_not_keepdims_random/", - "test_reduce_sum_empty_axes_input_noop_example/", - "test_reduce_sum_empty_axes_input_noop_random/", - "test_reduce_sum_keepdims_example/", - "test_reduce_sum_keepdims_random/", - "test_reduce_sum_negative_axes_keepdims_example/", - "test_reduce_sum_negative_axes_keepdims_random/", - "test_resize_downsample_sizes_cubic/", - "test_resize_downsample_sizes_linear_pytorch_half_pixel/", - "test_resize_downsample_sizes_nearest/", - "test_resize_tf_crop_and_resize/", - "test_resize_upsample_sizes_cubic/", - "test_resize_upsample_sizes_nearest/", - "test_resize_upsample_sizes_nearest_ceil_half_pixel/", - "test_resize_upsample_sizes_nearest_floor_align_corners/", - "test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric/", - "test_rnn_seq_length/", - "test_round/", - "test_scan9_sum/", - "test_scan_sum/", - "test_sce_NCd1_mean_weight_negative_ii/", - "test_sce_NCd1_mean_weight_negative_ii_expanded/", - "test_sce_NCd1_mean_weight_negative_ii_log_prob/", - "test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded/", - "test_sce_NCd1d2d3_none_no_weight_negative_ii/", - "test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded/", - "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob/", - "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded/", - "test_sce_NCd1d2d3_sum_weight_high_ii/", - "test_sce_NCd1d2d3_sum_weight_high_ii_expanded/", - "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob/", - "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded/", - "test_sce_NCd1d2d3d4d5_mean_weight/", - "test_sce_NCd1d2d3d4d5_mean_weight_expanded/", - "test_sce_NCd1d2d3d4d5_mean_weight_log_prob/", - "test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded/", - "test_sce_NCd1d2d3d4d5_none_no_weight/", - "test_sce_NCd1d2d3d4d5_none_no_weight_expanded/", - "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob/", - "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded/", - "test_sce_mean/", - "test_sce_mean_3d/", - "test_sce_mean_3d_expanded/", - "test_sce_mean_3d_log_prob/", - "test_sce_mean_3d_log_prob_expanded/", - "test_sce_mean_expanded/", - "test_sce_mean_log_prob/", - "test_sce_mean_log_prob_expanded/", - "test_sce_mean_no_weight_ii/", - "test_sce_mean_no_weight_ii_3d/", - "test_sce_mean_no_weight_ii_3d_expanded/", - "test_sce_mean_no_weight_ii_3d_log_prob/", - "test_sce_mean_no_weight_ii_3d_log_prob_expanded/", - "test_sce_mean_no_weight_ii_4d/", - "test_sce_mean_no_weight_ii_4d_expanded/", - "test_sce_mean_no_weight_ii_4d_log_prob/", - "test_sce_mean_no_weight_ii_4d_log_prob_expanded/", - "test_sce_mean_no_weight_ii_expanded/", - "test_sce_mean_no_weight_ii_log_prob/", - "test_sce_mean_no_weight_ii_log_prob_expanded/", - "test_sce_mean_weight/", - "test_sce_mean_weight_expanded/", - "test_sce_mean_weight_ii/", - "test_sce_mean_weight_ii_3d/", - "test_sce_mean_weight_ii_3d_expanded/", - "test_sce_mean_weight_ii_3d_log_prob/", - "test_sce_mean_weight_ii_3d_log_prob_expanded/", - "test_sce_mean_weight_ii_4d/", - "test_sce_mean_weight_ii_4d_expanded/", - "test_sce_mean_weight_ii_4d_log_prob/", - "test_sce_mean_weight_ii_4d_log_prob_expanded/", - "test_sce_mean_weight_ii_expanded/", - "test_sce_mean_weight_ii_log_prob/", - "test_sce_mean_weight_ii_log_prob_expanded/", - "test_sce_mean_weight_log_prob/", - "test_sce_mean_weight_log_prob_expanded/", - "test_sce_none/", - "test_sce_none_expanded/", - "test_sce_none_log_prob/", - "test_sce_none_log_prob_expanded/", - "test_sce_none_weights/", - "test_sce_none_weights_expanded/", - "test_sce_none_weights_log_prob/", - "test_sce_none_weights_log_prob_expanded/", - "test_sce_sum/", - "test_sce_sum_expanded/", - "test_sce_sum_log_prob/", - "test_sce_sum_log_prob_expanded/", - "test_sequence_insert_at_back/", - "test_sequence_insert_at_front/", - "test_simple_rnn_defaults/", - "test_simple_rnn_with_initial_bias/", - "test_softmax_axis_0/", - "test_softmax_axis_0_expanded/", - "test_softmax_axis_1/", - "test_softmax_axis_1_expanded/", - "test_softmax_axis_2_expanded/", - "test_softmax_default_axis/", - "test_softmax_default_axis_expanded/", - "test_softmax_example_expanded/", - "test_softmax_large_number_expanded/", - "test_softmax_negative_axis_expanded/", - "test_split_variable_parts_1d/", - "test_split_variable_parts_2d/", - "test_split_variable_parts_default_axis/", - "test_split_zero_size_splits/", - "test_squeeze/", - "test_squeeze_negative_axes/", - "test_strnormalizer_export_monday_casesensintive_lower/", - "test_strnormalizer_export_monday_casesensintive_nochangecase/", - "test_strnormalizer_export_monday_casesensintive_upper/", - "test_strnormalizer_export_monday_empty_output/", - "test_strnormalizer_export_monday_insensintive_upper_twodim/", - "test_strnormalizer_nostopwords_nochangecase/", - "test_tfidfvectorizer_tf_batch_onlybigrams_skip0/", - "test_tfidfvectorizer_tf_batch_onlybigrams_skip5/", - "test_tfidfvectorizer_tf_batch_uniandbigrams_skip5/", - "test_tfidfvectorizer_tf_only_bigrams_skip0/", - "test_tfidfvectorizer_tf_onlybigrams_levelempty/", - "test_tfidfvectorizer_tf_onlybigrams_skip5/", - "test_tfidfvectorizer_tf_uniandbigrams_skip5/", - "test_training_dropout/", - "test_training_dropout_default/", - "test_training_dropout_default_mask/", - "test_training_dropout_mask/", - "test_training_dropout_zero_ratio/", - "test_training_dropout_zero_ratio_mask/", - "test_unique_sorted_with_axis/", - "test_unique_sorted_with_axis_3d/", - "test_unique_sorted_with_negative_axis/", - "test_unsqueeze_axis_0/", - "test_unsqueeze_axis_1/", - "test_unsqueeze_axis_2/", - "test_unsqueeze_negative_axes/", - "test_unsqueeze_three_axes/", - "test_unsqueeze_two_axes/", - "test_unsqueeze_unsorted_axes/", - "test_upsample_nearest/", + "test_adagrad", + "test_adagrad_multiple", + "test_adam", + "test_adam_multiple", + "test_argmax_default_axis_example_select_last_index", + "test_argmax_default_axis_random_select_last_index", + "test_argmax_keepdims_example_select_last_index", + "test_argmax_keepdims_random_select_last_index", + "test_argmax_negative_axis_keepdims_example_select_last_index", + "test_argmax_negative_axis_keepdims_random_select_last_index", + "test_argmax_no_keepdims_example_select_last_index", + "test_argmax_no_keepdims_random_select_last_index", + "test_argmin_default_axis_example_select_last_index", + "test_argmin_default_axis_random_select_last_index", + "test_argmin_keepdims_example_select_last_index", + "test_argmin_keepdims_random_select_last_index", + "test_argmin_negative_axis_keepdims_example_select_last_index", + "test_argmin_negative_axis_keepdims_random_select_last_index", + "test_argmin_no_keepdims_example_select_last_index", + "test_argmin_no_keepdims_random_select_last_index", + "test_cast_BFLOAT16_to_FLOAT", + "test_cast_DOUBLE_to_FLOAT16", + "test_cast_FLOAT_to_BFLOAT16", + "test_cast_FLOAT_to_STRING", + "test_cast_STRING_to_FLOAT", + "test_celu", + "test_compress_0", + "test_compress_1", + "test_compress_default_axis", + "test_compress_negative_axis", + "test_convtranspose_dilations", + "test_convtranspose_output_shape", + "test_cumsum_1d", + "test_cumsum_1d_exclusive", + "test_cumsum_1d_reverse", + "test_cumsum_1d_reverse_exclusive", + "test_cumsum_2d_axis_0", + "test_cumsum_2d_axis_1", + "test_cumsum_2d_negative_axis", + "test_det_2d", + "test_det_nd", + "test_dropout_default", + "test_dropout_default_mask", + "test_dropout_default_mask_ratio", + "test_dropout_default_ratio", + "test_einsum_batch_diagonal", + "test_einsum_batch_matmul", + "test_einsum_inner_prod", + "test_einsum_sum", + "test_einsum_transpose", + "test_greater_equal", + "test_greater_equal_bcast", + "test_hardmax_axis_0", + "test_hardmax_axis_1", + "test_hardmax_default_axis", + "test_if_seq", + "test_less_equal", + "test_less_equal_bcast", + "test_logsoftmax_axis_0", + "test_logsoftmax_axis_0_expanded", + "test_logsoftmax_axis_1", + "test_logsoftmax_axis_1_expanded", + "test_logsoftmax_axis_2_expanded", + "test_logsoftmax_default_axis", + "test_logsoftmax_default_axis_expanded", + "test_logsoftmax_example_1_expanded", + "test_logsoftmax_large_number_expanded", + "test_logsoftmax_negative_axis_expanded", + "test_loop11", + "test_loop13_seq", + "test_matmulinteger", + "test_maxpool_2d_same_lower", + "test_maxpool_2d_same_upper", + "test_maxpool_with_argmax_2d_precomputed_pads", + "test_maxpool_with_argmax_2d_precomputed_strides", + "test_maxunpool_export_with_output_shape", + "test_momentum", + "test_momentum_multiple", + "test_mvn", + "test_nesterov_momentum", + "test_nllloss_NC", + "test_nllloss_NC_expanded", + "test_nllloss_NCd1", + "test_nllloss_NCd1_expanded", + "test_nllloss_NCd1_ii", + "test_nllloss_NCd1_ii_expanded", + "test_nllloss_NCd1_mean_weight_negative_ii", + "test_nllloss_NCd1_mean_weight_negative_ii_expanded", + "test_nllloss_NCd1_weight", + "test_nllloss_NCd1_weight_expanded", + "test_nllloss_NCd1_weight_ii", + "test_nllloss_NCd1_weight_ii_expanded", + "test_nllloss_NCd1d2", + "test_nllloss_NCd1d2_expanded", + "test_nllloss_NCd1d2_no_weight_reduction_mean_ii", + "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", + "test_nllloss_NCd1d2_reduction_mean", + "test_nllloss_NCd1d2_reduction_mean_expanded", + "test_nllloss_NCd1d2_reduction_sum", + "test_nllloss_NCd1d2_reduction_sum_expanded", + "test_nllloss_NCd1d2_with_weight", + "test_nllloss_NCd1d2_with_weight_expanded", + "test_nllloss_NCd1d2_with_weight_reduction_mean", + "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", + "test_nllloss_NCd1d2_with_weight_reduction_sum", + "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", + "test_nllloss_NCd1d2_with_weight_reduction_sum_ii", + "test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", + "test_nllloss_NCd1d2d3_none_no_weight_negative_ii", + "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", + "test_nllloss_NCd1d2d3_sum_weight_high_ii", + "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", + "test_nllloss_NCd1d2d3d4d5_mean_weight", + "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", + "test_nllloss_NCd1d2d3d4d5_none_no_weight", + "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", + "test_pow_types_float", + "test_pow_types_float32_int32", + "test_pow_types_float32_int64", + "test_pow_types_float32_uint32", + "test_pow_types_float32_uint64", + "test_pow_types_int", + "test_pow_types_int32_float32", + "test_pow_types_int32_int32", + "test_pow_types_int64_float32", + "test_pow_types_int64_int64", + "test_qlinearmatmul_2D", + "test_qlinearmatmul_3D", + "test_range_float_type_positive_delta_expanded", + "test_range_int32_type_negative_delta_expanded", + "test_reduce_sum_default_axes_keepdims_example", + "test_reduce_sum_default_axes_keepdims_random", + "test_reduce_sum_do_not_keepdims_example", + "test_reduce_sum_do_not_keepdims_random", + "test_reduce_sum_empty_axes_input_noop_example", + "test_reduce_sum_empty_axes_input_noop_random", + "test_reduce_sum_keepdims_example", + "test_reduce_sum_keepdims_random", + "test_reduce_sum_negative_axes_keepdims_example", + "test_reduce_sum_negative_axes_keepdims_random", + "test_resize_downsample_sizes_cubic", + "test_resize_downsample_sizes_linear_pytorch_half_pixel", + "test_resize_downsample_sizes_nearest", + "test_resize_tf_crop_and_resize", + "test_resize_upsample_sizes_cubic", + "test_resize_upsample_sizes_nearest", + "test_resize_upsample_sizes_nearest_ceil_half_pixel", + "test_resize_upsample_sizes_nearest_floor_align_corners", + "test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric", + "test_rnn_seq_length", + "test_round", + "test_scan9_sum", + "test_scan_sum", + "test_sce_NCd1_mean_weight_negative_ii", + "test_sce_NCd1_mean_weight_negative_ii_expanded", + "test_sce_NCd1_mean_weight_negative_ii_log_prob", + "test_sce_NCd1_mean_weight_negative_ii_log_prob_expanded", + "test_sce_NCd1d2d3_none_no_weight_negative_ii", + "test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", + "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", + "test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", + "test_sce_NCd1d2d3_sum_weight_high_ii", + "test_sce_NCd1d2d3_sum_weight_high_ii_expanded", + "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", + "test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", + "test_sce_NCd1d2d3d4d5_mean_weight", + "test_sce_NCd1d2d3d4d5_mean_weight_expanded", + "test_sce_NCd1d2d3d4d5_mean_weight_log_prob", + "test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", + "test_sce_NCd1d2d3d4d5_none_no_weight", + "test_sce_NCd1d2d3d4d5_none_no_weight_expanded", + "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", + "test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", + "test_sce_mean", + "test_sce_mean_3d", + "test_sce_mean_3d_expanded", + "test_sce_mean_3d_log_prob", + "test_sce_mean_3d_log_prob_expanded", + "test_sce_mean_expanded", + "test_sce_mean_log_prob", + "test_sce_mean_log_prob_expanded", + "test_sce_mean_no_weight_ii", + "test_sce_mean_no_weight_ii_3d", + "test_sce_mean_no_weight_ii_3d_expanded", + "test_sce_mean_no_weight_ii_3d_log_prob", + "test_sce_mean_no_weight_ii_3d_log_prob_expanded", + "test_sce_mean_no_weight_ii_4d", + "test_sce_mean_no_weight_ii_4d_expanded", + "test_sce_mean_no_weight_ii_4d_log_prob", + "test_sce_mean_no_weight_ii_4d_log_prob_expanded", + "test_sce_mean_no_weight_ii_expanded", + "test_sce_mean_no_weight_ii_log_prob", + "test_sce_mean_no_weight_ii_log_prob_expanded", + "test_sce_mean_weight", + "test_sce_mean_weight_expanded", + "test_sce_mean_weight_ii", + "test_sce_mean_weight_ii_3d", + "test_sce_mean_weight_ii_3d_expanded", + "test_sce_mean_weight_ii_3d_log_prob", + "test_sce_mean_weight_ii_3d_log_prob_expanded", + "test_sce_mean_weight_ii_4d", + "test_sce_mean_weight_ii_4d_expanded", + "test_sce_mean_weight_ii_4d_log_prob", + "test_sce_mean_weight_ii_4d_log_prob_expanded", + "test_sce_mean_weight_ii_expanded", + "test_sce_mean_weight_ii_log_prob", + "test_sce_mean_weight_ii_log_prob_expanded", + "test_sce_mean_weight_log_prob", + "test_sce_mean_weight_log_prob_expanded", + "test_sce_none", + "test_sce_none_expanded", + "test_sce_none_log_prob", + "test_sce_none_log_prob_expanded", + "test_sce_none_weights", + "test_sce_none_weights_expanded", + "test_sce_none_weights_log_prob", + "test_sce_none_weights_log_prob_expanded", + "test_sce_sum", + "test_sce_sum_expanded", + "test_sce_sum_log_prob", + "test_sce_sum_log_prob_expanded", + "test_sequence_insert_at_back", + "test_sequence_insert_at_front", + "test_simple_rnn_defaults", + "test_simple_rnn_with_initial_bias", + "test_softmax_axis_0", + "test_softmax_axis_0_expanded", + "test_softmax_axis_1", + "test_softmax_axis_1_expanded", + "test_softmax_axis_2_expanded", + "test_softmax_default_axis", + "test_softmax_default_axis_expanded", + "test_softmax_example_expanded", + "test_softmax_large_number_expanded", + "test_softmax_negative_axis_expanded", + "test_split_variable_parts_1d", + "test_split_variable_parts_2d", + "test_split_variable_parts_default_axis", + "test_split_zero_size_splits", + "test_squeeze", + "test_squeeze_negative_axes", + "test_strnormalizer_export_monday_casesensintive_lower", + "test_strnormalizer_export_monday_casesensintive_nochangecase", + "test_strnormalizer_export_monday_casesensintive_upper", + "test_strnormalizer_export_monday_empty_output", + "test_strnormalizer_export_monday_insensintive_upper_twodim", + "test_strnormalizer_nostopwords_nochangecase", + "test_tfidfvectorizer_tf_batch_onlybigrams_skip0", + "test_tfidfvectorizer_tf_batch_onlybigrams_skip5", + "test_tfidfvectorizer_tf_batch_uniandbigrams_skip5", + "test_tfidfvectorizer_tf_only_bigrams_skip0", + "test_tfidfvectorizer_tf_onlybigrams_levelempty", + "test_tfidfvectorizer_tf_onlybigrams_skip5", + "test_tfidfvectorizer_tf_uniandbigrams_skip5", + "test_training_dropout", + "test_training_dropout_default", + "test_training_dropout_default_mask", + "test_training_dropout_mask", + "test_training_dropout_zero_ratio", + "test_training_dropout_zero_ratio_mask", + "test_unique_sorted_with_axis", + "test_unique_sorted_with_axis_3d", + "test_unique_sorted_with_negative_axis", + "test_unsqueeze_axis_0", + "test_unsqueeze_axis_1", + "test_unsqueeze_axis_2", + "test_unsqueeze_negative_axes", + "test_unsqueeze_three_axes", + "test_unsqueeze_two_axes", + "test_unsqueeze_unsorted_axes", + "test_upsample_nearest", ] target_skips = { "cuda": [ - "test_basic_convinteger/", - "test_convinteger_with_padding/", - "test_mod_mixed_sign_float16/", - "test_qlinearconv/", - "test_resize_upsample_sizes_nearest/", + "test_basic_convinteger", + "test_convinteger_with_padding", + "test_range_float_type_positive_delta_expanded", + "test_range_int32_type_positive_delta_expanded", + "test_mod_mixed_sign_float16", + "test_qlinearconv", + "test_resize_upsample_sizes_nearest", ] } -@pytest.mark.parametrize("test", onnx_test_folders) +@pytest.mark.parametrize("onnx_test", onnx_test_folders) @tvm.testing.parametrize_targets -def test_onnx_nodes(test, target, dev): - if target in target_skips: - for failure in target_skips[target]: - if failure in test: - pytest.skip() - break - for failure in unsupported_onnx_tests: - if failure in test: - pytest.skip() - break +def test_onnx_nodes(target, dev, onnx_test): + target_kind = tvm.target.Target(target).kind.name + + if onnx_test in unsupported_onnx_tests: + pytest.skip(f"Onnx test '{onnx_test}' not yet supported by TVM") + + target_specific_skips = target_skips.get(target_kind, []) + if onnx_test in target_specific_skips: + pytest.skip(f"Onnx test '{onnx_test}' not yet supported by TVM on {target_kind} targets") + + test_dir = os.path.join(onnx_test_node_dir, onnx_test) + atol = 1e-5 rtol = 1e-5 - if "roialign" in test: + if "roialign" in test_dir: # for some reason the ONNX test crops the # roialign results to 4 decimal places atol = 1e-4 - onnx_model = onnx.load(test + "/model.onnx") + onnx_model = onnx.load(test_dir + "/model.onnx") inputs = [] outputs = [] - for dataset in glob.glob(test + "/*/"): + for dataset in glob.glob(test_dir + "/*/"): tensors = sorted(glob.glob(dataset + "/*.pb")) for tensor in tensors: new_tensor = onnx.TensorProto() @@ -5068,11 +5075,9 @@ def verify_reverse_sequence(x, sequence_lens, batch_axis, time_axis): verify_reverse_sequence(x, sequence_lens, 1, 0) +@tvm.testing.known_failing_targets("cuda") @tvm.testing.parametrize_targets def test_qlinearconv(target, dev): - if "cuda" in target: - pytest.skip("Fails on CUDA") - def verify_qlinearconv( x_shape, w_shape, @@ -5370,11 +5375,9 @@ def get_random_uniform(shape, dtype="float32", high=1.0, low=0.0, seed=None): tvm.testing.assert_allclose(real, expected, rtol=1e-5) +@tvm.testing.known_failing_targets("cuda") @tvm.testing.parametrize_targets def test_convinteger(target, dev): - if "cuda" in target: - pytest.skip("Fails on CUDA") - def verify_convinteger( x_shape, w_shape, diff --git a/tests/python/topi/python/test_topi_dense.py b/tests/python/topi/python/test_topi_dense.py index 964c1621fa47..235a09400387 100644 --- a/tests/python/topi/python/test_topi_dense.py +++ b/tests/python/topi/python/test_topi_dense.py @@ -135,7 +135,6 @@ def test_dense( @pytest.mark.parametrize("target,in_dtype,out_dtype", [("cuda", "int8", "int32")]) -@tvm.testing.requires_gpu def test_dense_cuda_int8( target, dev, diff --git a/tests/python/unittest/test_tvm_testing_features.py b/tests/python/unittest/test_tvm_testing_features.py index 07b8c652bf1f..4b699096e96a 100644 --- a/tests/python/unittest/test_tvm_testing_features.py +++ b/tests/python/unittest/test_tvm_testing_features.py @@ -180,5 +180,35 @@ def test_num_uses_cached(self): assert self.num_uses_broken_cached_fixture == 0 +class TestAutomaticMarks: + @staticmethod + def check_marks(request, target): + parameter = tvm.testing._pytest_target_params([target])[0] + required_marks = [decorator.mark for decorator in parameter.marks] + applied_marks = list(request.node.iter_markers()) + + for required_mark in required_marks: + assert required_mark in applied_marks + + def test_automatic_fixture(self, request, target): + self.check_marks(request, target) + + @tvm.testing.parametrize_targets + def test_bare_parametrize(self, request, target): + self.check_marks(request, target) + + @tvm.testing.parametrize_targets("llvm", "cuda", "vulkan") + def test_explicit_parametrize(self, request, target): + self.check_marks(request, target) + + @pytest.mark.parametrize("target", ["llvm", "cuda", "vulkan"]) + def test_pytest_mark(self, request, target): + self.check_marks(request, target) + + @pytest.mark.parametrize("target,other_param", [("llvm", 0), ("cuda", 1), ("vulkan", 2)]) + def test_pytest_mark_covariant(self, request, target, other_param): + self.check_marks(request, target) + + if __name__ == "__main__": sys.exit(pytest.main(sys.argv))