Activate everywhere

pytest.ini

@@ -23,6 +23,7 @@ markers =
     gpu: mark a test that requires GPU.
     integration: mark an integration test
     onnx_coverage: ONNX coverage test
+    garbage_expected: this teast leaks ndarray references. The tested functionality is broken.
 
 env =
     MXNET_HOME=tests/data

tests/python/conftest.py

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""conftest.py contains configuration for pytest."""
+
+import gc
+
+import mxnet as mx
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def check_leak_ndarray(request):
+    garbage_expected = request.node.get_closest_marker('garbage_expected')
+    if garbage_expected:  # Some tests leak references. They should be fixed.
+        yield  # run test
+        return
+
+    del gc.garbage[:]
+    # Collect garbage prior to running the next test
+    gc.collect()
+    # Enable gc debug mode to check if the test leaks any arrays
+    gc_flags = gc.get_debug()
+    gc.set_debug(gc.DEBUG_SAVEALL)
+
+    # Run the test
+    yield
+
+    # Check for leaked NDArrays
+    gc.collect()
+    gc.set_debug(gc_flags)  # reset gc flags
+
+    seen = set()
+    def has_array(element):
+        try:
+            if element in seen:
+                return False
+            seen.add(element)
+        except TypeError:  # unhashable
+            pass
+
+        if isinstance(element, mx.nd._internal.NDArrayBase):
+            return True
+        elif isinstance(element, mx.sym._internal.SymbolBase):
+            return False
+        elif hasattr(element, '__dict__'):
+            return any(has_array(x) for x in vars(element))
+        elif isinstance(element, dict):
+            return any(has_array(x) for x in element.items())
+        else:
+            try:
+                return any(has_array(x) for x in element)
+            except (TypeError, KeyError, RecursionError):
+                return False
+
+    assert not any(has_array(x) for x in gc.garbage), 'Found leaked NDArrays due to reference cycles'
+    del gc.garbage[:]

tests/python/unittest/test_autograd.py

@@ -23,6 +23,8 @@
 from common import setup_module, with_seed, teardown_module, xfail_when_nonstandard_decimal_separator
 from mxnet.test_utils import EnvManager
 
+import pytest
+
 
 def grad_and_loss(func, argnum=None):
     """Return function that computes both gradient of arguments and loss value.
@@ -344,6 +346,7 @@ def test_is_train():
         assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_function():
     class func(Function):
         def forward(self, x, y):

tests/python/unittest/test_gluon.py

@@ -38,49 +38,6 @@
 import random
 import tempfile
 
-@pytest.fixture(autouse=True)
-def check_leak_ndarray():
-    del gc.garbage[:]
-    # Collect garbage prior to running the next test
-    gc.collect()
-    # Enable gc debug mode to check if the test leaks any arrays
-    gc_flags = gc.get_debug()
-    gc.set_debug(gc.DEBUG_SAVEALL)
-
-    # Run the test
-    yield
-
-    # Check for leaked NDArrays
-    gc.collect()
-    gc.set_debug(gc_flags)  # reset gc flags
-
-    seen = set()
-    def has_array(element):
-        try:
-            if element in seen:
-                return False
-            seen.add(element)
-        except TypeError:  # unhashable
-            pass
-
-        if isinstance(element, mx.nd._internal.NDArrayBase):
-            return True
-        elif isinstance(element, mx.sym._internal.SymbolBase):
-            return False
-        elif hasattr(element, '__dict__'):
-            return any(has_array(x) for x in vars(element))
-        elif isinstance(element, dict):
-            return any(has_array(x) for x in element.items())
-        else:
-            try:
-                return any(has_array(x) for x in element)
-            except (TypeError, KeyError, RecursionError):
-                return False
-
-    assert not any(has_array(x) for x in gc.garbage), 'Found leaked NDArrays due to reference cycles'
-    del gc.garbage[:]
-
-
 @with_seed()
 def test_parameter():
     p = gluon.Parameter('weight', shape=(10, 10))

tests/python/unittest/test_gluon_data.py

@@ -227,6 +227,7 @@ def test_image_list_dataset_handle():
         assert label == 0
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_list_dataset():
     for num_worker in range(0, 3):
         data = mx.gluon.data.DataLoader([([1,2], 0), ([3, 4], 1)], batch_size=1, num_workers=num_worker)
@@ -504,6 +505,7 @@ def test_dataset_take_handle():
         total += sample
     assert total == expected_total
 
+@pytest.mark.garbage_expected
 def test_dataloader_scope():
     """
     Bug: Gluon DataLoader terminates the process pool early while

tests/python/unittest/test_gluon_rnn.py

@@ -154,6 +154,7 @@ def test_lstmp():
 
 
 @pytest.mark.serial
+@pytest.mark.garbage_expected
 def test_lstm_forget_bias():
     forget_bias = 2.0
     stack = gluon.rnn.SequentialRNNCell()
@@ -196,6 +197,7 @@ def test_lstm_cpu_inference():
                                       rtol=1e-3, atol=1e-5)
 
 
+@pytest.mark.garbage_expected
 def test_gru():
     cell = gluon.rnn.GRUCell(100, prefix='rnn_', activation='relu', recurrent_activation='tanh')
     inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
@@ -787,6 +789,7 @@ def test_fused_lstm_layer():
 
 @with_seed()
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
+@pytest.mark.garbage_expected
 def test_fused_gru_layer():
     input_sizes = [8]
     hidden_sizes = [8, 16]
@@ -799,6 +802,7 @@ def test_fused_gru_layer():
 
 @with_seed()
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
+@pytest.mark.garbage_expected
 def test_fused_rnnrelu_layer():
     input_sizes = [8]
     hidden_sizes = [8, 16]
@@ -811,6 +815,7 @@ def test_fused_rnnrelu_layer():
 
 @with_seed()
 @assert_raises_cudnn_not_satisfied(min_version='5.1.10')
+@pytest.mark.garbage_expected
 def test_fused_rnntanh_layer():
     input_sizes = [8]
     hidden_sizes = [8, 16]
@@ -822,6 +827,7 @@ def test_fused_rnntanh_layer():
 
 
 @pytest.mark.serial
+@pytest.mark.garbage_expected
 def test_rnn_unroll_variant_length():
     # Test for imperative usage
     cell_list = []
@@ -896,6 +902,7 @@ def test_rnn_unroll_variant_length():
     mod.get_outputs()[0].asnumpy()
 
 
+@pytest.mark.garbage_expected
 def test_cell_fill_shape():
     cell = gluon.rnn.LSTMCell(10, input_size=7)
     cell.hybridize()

tests/python/unittest/test_module.py

@@ -212,6 +212,7 @@ def dict_equ(a, b):
 
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_bucketing_save_load(tmpdir):
     previous_update_on_kvstore = os.getenv('MXNET_UPDATE_ON_KVSTORE', "1")
     os.putenv('MXNET_UPDATE_ON_KVSTORE', '1')
@@ -462,6 +463,7 @@ def test_module_set_params():
 
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_monitor():
     # data iter
     data = mx.nd.array([[0.05, .10]]);

tests/python/unittest/test_numpy_interoperability.py

@@ -3256,6 +3256,7 @@ def test_np_memory_array_function():
 @use_np
 @with_array_function_protocol
 @pytest.mark.serial
+@pytest.mark.garbage_expected
 def test_np_array_function_protocol():
     check_interoperability(_NUMPY_ARRAY_FUNCTION_LIST)
 

tests/python/unittest/test_operator.py

@@ -720,6 +720,7 @@ def check_softmax_with_shape(shape, xpu, preserve_shape=False):
     assert_almost_equal(grad, np_softmax(x.asnumpy()) - l.asnumpy(), rtol=rtol, atol=atol)
 
 
+@pytest.mark.garbage_expected
 def test_python_op():
     X = mx.symbol.Variable('X')
     op = mx.operator.NumpyOp()
@@ -5820,6 +5821,7 @@ def test_rcbrt_op():
 
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_custom_op():
     class Sqr(mx.operator.CustomOp):
         def forward(self, is_train, req, in_data, out_data, aux):
@@ -6081,6 +6083,7 @@ def create_operator(self, ctx, shapes, dtypes):
             return Dot()
 
 @with_seed()
+@pytest.mark.garbage_expected
 def test_custom_op_exc():
     # test except handling
     # see https://github.com/apache/incubator-mxnet/pull/14693

tests/python/unittest/test_profiler.py

@@ -77,6 +77,7 @@ def test_profiler():
     profiler.set_state('stop')
 
 
+@pytest.mark.garbage_expected
 def test_profile_create_domain():
     enable_profiler('test_profile_create_domain.json')
     domain = profiler.Domain(name='PythonDomain')
@@ -304,6 +305,7 @@ def test_aggregate_duplication():
     profiler.set_state('stop')
 
 
+@pytest.mark.garbage_expected
 def test_custom_operator_profiling(seed=None, file_name=None):
     class Sigmoid(mx.operator.CustomOp):
         def forward(self, is_train, req, in_data, out_data, aux):

tests/python/unittest/test_subgraph.py

@@ -140,6 +140,7 @@ def make_subgraph4(stype):
 
 
 @pytest.mark.serial
+@pytest.mark.garbage_expected
 def test_subgraph_with_customOp():
     class MyAdd(mx.operator.CustomOp):
         def forward(self, is_train, req, in_data, out_data, aux):