Merge pull request #490 from glatterf42/enh/ci/flaky-tests

Handling flaky tests

ixmp/backend/jdbc.py

@@ -385,7 +385,7 @@ def get_log_level(self):
     def set_doc(self, domain, docs):
         dd = _domain_enum(domain)
         jdata = java.LinkedHashMap()
-        if type(docs) == dict:
+        if isinstance(docs, dict):
             docs = list(docs.items())
         for k, v in docs:
             jdata.put(str(k), str(v))

ixmp/tests/backend/test_jdbc.py

@@ -1,5 +1,7 @@
 import gc
 import logging
+import os
+import platform
 from sys import getrefcount
 from typing import Tuple
 
@@ -17,6 +19,12 @@
 log = logging.getLogger(__name__)
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Linux",
+    reason="Flaky; see iiasa/ixmp#489",
+)
 def test_jvm_warn(recwarn):
     """Test that no warnings are issued on JVM start-up.
 
@@ -38,6 +46,12 @@ def test_jvm_warn(recwarn):
         assert len(recwarn) == 0, recwarn.pop().message
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Windows",
+    reason="Flaky; see iiasa/ixmp#489",
+)
 def test_close(test_mp_f, capfd):
     """Platform.close_db() doesn't throw needless exceptions."""
     # Use the session-scoped fixture to avoid affecting other tests in this file
@@ -176,6 +190,12 @@ def test_invalid_properties_file(test_data_path):
         ixmp.Platform(dbprops=test_data_path / "hsqldb.properties")
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Windows",
+    reason="Flaky; see iiasa/ixmp#489",
+)
 def test_connect_message(capfd, caplog):
     msg = "connected to database 'jdbc:hsqldb:mem://ixmptest' (user: ixmp)..."
 

ixmp/tests/test_access.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import platform
 import sys
 from subprocess import Popen
 from time import sleep
@@ -50,6 +52,12 @@ def mock(server):
     yield httpmock
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/ixmp#489",
+)
 def test_check_single_model_access(mock, tmp_path, test_data_path):
     mock.when(
         "POST /access/list",

ixmp/tests/test_integration.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import platform
 
 import numpy as np
 import pytest
@@ -93,6 +95,12 @@ def get_distance(scen):
     return scen.par("d").set_index(["i", "j"]).loc["san-diego", "topeka"]["value"]
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/ixmp#489",
+)
 def test_multi_db_run(tmpdir):
     # create a new instance of the transport problem and solve it
     mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1")

ixmp/tests/test_tutorials.py

@@ -1,12 +1,21 @@
 import os
+import platform
 import sys
 
 import numpy as np
 import pytest
 
 from ixmp.testing import get_cell_output, run_notebook
 
+FLAKY = pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/ixmp#489",
+)
+
 
+@FLAKY
 def test_py_transport(tutorial_path, tmp_path, tmp_env):
     fname = tutorial_path / "transport" / "py_transport.ipynb"
     nb, errors = run_notebook(fname, tmp_path, tmp_env)
@@ -16,6 +25,7 @@ def test_py_transport(tutorial_path, tmp_path, tmp_env):
     assert np.isclose(get_cell_output(nb, -5)["lvl"], 153.6750030517578)
 
 
+@FLAKY
 def test_py_transport_scenario(tutorial_path, tmp_path, tmp_env):
     fname = tutorial_path / "transport" / "py_transport_scenario.ipynb"
     nb, errors = run_notebook(fname, tmp_path, tmp_env)
@@ -25,6 +35,7 @@ def test_py_transport_scenario(tutorial_path, tmp_path, tmp_env):
     assert np.isclose(get_cell_output(nb, "scen-detroit-z")["lvl"], 161.324)
 
 
+@FLAKY
 @pytest.mark.rixmp
 # TODO investigate and resolve the cause of the time outs; remove this mark
 @pytest.mark.skipif(
@@ -36,6 +47,7 @@ def test_R_transport(tutorial_path, tmp_path, tmp_env):
     assert errors == []
 
 
+@FLAKY
 @pytest.mark.rixmp
 # TODO investigate and resolve the cause of the time outs; remove this mark
 @pytest.mark.skipif(

pyproject.toml

@@ -68,6 +68,7 @@ tests = [
   "pytest >= 5",
   "pytest-benchmark",
   "pytest-cov",
+  "pytest-rerunfailures",
 ]
 
 [project.scripts]