Add retries to flaky galaxy tasks (#1431)

.pre-commit-config.yaml

@@ -80,6 +80,7 @@ repos:
     additional_dependencies:
     - Sphinx>=3.1.2
     - enrich
+    - flaky
     - yamllint
     - wcmatch
 - repo: https://github.com/pre-commit/mirrors-pylint
@@ -92,6 +93,7 @@ repos:
     - ansible-base
     - docutils
     - enrich
+    - flaky
     - pytest
     - pyyaml
     - rich

docs/requirements.txt

@@ -32,7 +32,7 @@ pyparsing==2.4.7
 pytz==2021.1
 pyyaml==5.4.1
 requests==2.25.1
-rich==9.11.1
+rich==9.12.4
 snowballstemmer==2.1.0
 sphinx-ansible-theme==0.3.2
 sphinx-notfound-page==0.6

mypy.ini

@@ -15,6 +15,10 @@ ignore_missing_imports = True
 [mypy-ansible.*]
 ignore_missing_imports = True
 
+[mypy-flaky]
+# # https://github.com/box/flaky/issues/170
+ignore_missing_imports = True
+
 [mypy-pytest]
 ignore_missing_imports = True
 

pytest.ini

@@ -7,6 +7,9 @@ addopts =
     # useful for live testing with `pytest-watch` during development:
     # --testmon
 
+    # flaky:
+    --no-success-flaky-report
+
     --durations=10
     -v
     -ra

test-requirements.in

@@ -2,6 +2,7 @@
 # Avoid using --generate-hashes as it breaks pip install from tox with:
 # ERROR: In --require-hashes mode, all requirements must have their versions pinned with ==. These do not:
 #    ansible<2.10,>=2.9 from
+flaky >= 3.7.0
 pytest >= 6.0.1
 pytest-cov >= 2.10.1
 pytest-xdist >= 2.1.0

test-requirements.txt

@@ -9,9 +9,10 @@ attrs==20.3.0
 bracex==2.1.1
 colorama==0.4.4
 commonmark==0.9.1
-coverage==5.4
+coverage==5.5
 enrich==1.2.6
 execnet==1.8.0
+flaky==3.7.0
 iniconfig==1.1.1
 packaging==20.9
 pluggy==0.13.1
@@ -23,7 +24,7 @@ pytest-forked==1.3.0
 pytest-xdist==2.2.1
 pytest==6.2.2
 pyyaml==5.4.1
-rich==9.11.1
+rich==9.12.4
 toml==0.10.2
 typing-extensions==3.7.4.3
 wcmatch==8.1.1

test/test_prerun.py

@@ -1,9 +1,13 @@
 """Tests related to prerun part of the linter."""
 import os
 
+from flaky import flaky
+
 from ansiblelint.testing import run_ansible_lint
 
 
+# https://github.com/box/flaky/issues/170
+@flaky(max_runs=3)  # type: ignore
 def test_prerun_reqs_v1() -> None:
     """Checks that the linter can auto-install requirements v1 when found."""
     cwd = os.path.realpath(
@@ -12,11 +16,14 @@ def test_prerun_reqs_v1() -> None:
         )
     )
     result = run_ansible_lint(".", cwd=cwd)
-    assert "Running ansible-galaxy role install" in result.stderr
-    assert "Running ansible-galaxy collection install" not in result.stderr
-    assert result.returncode == 0
+    assert "Running ansible-galaxy role install" in result.stderr, result.stderr
+    assert (
+        "Running ansible-galaxy collection install" not in result.stderr
+    ), result.stderr
+    assert result.returncode == 0, result
 
 
+@flaky(max_runs=3)  # type: ignore
 def test_prerun_reqs_v2() -> None:
     """Checks that the linter can auto-install requirements v2 when found."""
     cwd = os.path.realpath(
@@ -25,6 +32,6 @@ def test_prerun_reqs_v2() -> None:
         )
     )
     result = run_ansible_lint(".", cwd=cwd)
-    assert "Running ansible-galaxy role install" in result.stderr
-    assert "Running ansible-galaxy collection install" in result.stderr
-    assert result.returncode == 0
+    assert "Running ansible-galaxy role install" in result.stderr, result.stderr
+    assert "Running ansible-galaxy collection install" in result.stderr, result.stderr
+    assert result.returncode == 0, result

tox.ini

@@ -80,6 +80,7 @@ passenv =
 description = Bump all test depeendencies
 basepython = python3
 deps =
+  pre-commit>=2.6.0
   pip-tools>=5.4.0
   setuptools>=51.1.1
 skip_install = true