pytest-rerunfailures in goth tests (#643)

This is an experimental feature, because
* we're not sure if this will solve problems with flaky tests
* there is an ugly fix in conftest.py

If this turns out useful, we might consider replacing pytest-rerunfailures with
a similar mechanism implemented inside goth.

pyproject.toml

@@ -57,6 +57,7 @@ integration-tests = ['goth', 'pytest', 'pytest-asyncio']
 [tool.poetry.dev-dependencies]
 black = "^21.7b0"
 pytest = "^6.2"
+pytest-rerunfailures = "^10.1"
 portray = "^1"
 pytest-asyncio = "^0.14"
 mypy = "^0.782"
@@ -109,7 +110,7 @@ ya-market = "0.1.0"
 [tool.poe.tasks]
 test = "pytest --cov=yapapi --ignore tests/goth_tests"
 goth-assets = "python -m goth create-assets tests/goth_tests/assets"
-goth-tests = "pytest -svx tests/goth_tests"
+goth-tests = "pytest -svx tests/goth_tests --reruns 3 --only-rerun AssertionError --only-rerun TimeoutError --only-rerun goth.runner.exceptions.TemporalAssertionError --only-rerun urllib.error.URLError --only-rerun goth.runner.exceptions.CommandError"
 typecheck = "mypy ."
 codestyle = "black --check --diff ."
 _liccheck_export = "poetry export -E cli -f requirements.txt -o .requirements.txt"

tests/goth_tests/conftest.py

@@ -1,3 +1,4 @@
+import asyncio
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import cast, List
@@ -8,6 +9,32 @@
 from yapapi.package import vm
 
 
+#  `pytest-rerunfailures` and `pytest-asyncio` don't work together
+#   (https://github.com/pytest-dev/pytest-rerunfailures/issues/154)
+#   The same problem occurs when `flaky` is used instead of `pytest-rerunfailures`.
+#   Here we have a patch that is quite ugly, but hopefully harmless.
+class LoopThatIsNeverClosed(asyncio.AbstractEventLoop):
+    """Just a loop, but if you try to use it after it was closed you use a fresh loop"""
+
+    def __init__(self):
+        self._loop = None
+
+    def __getattribute__(self, name):
+        if name == "_loop":
+            return super().__getattribute__("_loop")
+        if self._loop is None or self._loop._closed:
+            self._loop = asyncio.new_event_loop()
+        return getattr(self._loop, name)
+
+
+@pytest.fixture
+def event_loop():
+    """This overrides `pytest.asyncio` fixture"""
+    loop = LoopThatIsNeverClosed()
+    yield loop
+    loop.close()
+
+
 def pytest_addoption(parser):
     """Add optional parameters to pytest CLI invocations."""