ci: reruns failed tests (#1348)

* ci: rerun failed
* rename
* ellipse

.github/actions/unittesting/action.yml

@@ -57,7 +57,16 @@ runs:
 
   - name: Unittests
     working-directory: ./tests
-    run: python -m pytest -v --maxfail=5 ${{ inputs.dirs }} --cov=torchmetrics --junitxml="$PYTEST_ARTEFACT.xml" --durations=50 ${{ inputs.test-timeout }}
+    run: |
+      python -m pytest -v \
+        --maxfail=5 \
+        ${{ inputs.dirs }} \
+        --cov=torchmetrics \
+        --durations=50 \
+        --reruns 3 \
+        --reruns-delay 1 \
+        --junitxml="$PYTEST_ARTEFACT.xml" \
+        ${{ inputs.test-timeout }}
     shell: ${{ inputs.shell-type }}
 
   - name: Upload pytest results

.github/workflows/ci_integrate.yml → .github/workflows/ci-integrate.yml

@@ -75,4 +75,4 @@ jobs:
       working-directory: ./tests
       env:
         PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: 'python'
-      run: python -m pytest integrations -v --durations=25
+      run: python -m pytest integrations -v --durations=25 --reruns 3 --reruns-delay 2

requirements/test.txt

@@ -3,6 +3,7 @@ pytest==6.*
 pytest-cov>2.10
 # pytest-flake8
 pytest-doctestplus>=0.9.0
+pytest-rerunfailures>=10.0
 pytest-timeout
 check-manifest
 phmdoctest>=1.1.1

src/torchmetrics/utilities/checks.py

@@ -647,11 +647,11 @@ def check_forward_full_state_property(
 
     Example (states in ``update`` are independent, save to set ``full_state_update=False``)
         >>> from torchmetrics import ConfusionMatrix
-        >>> check_forward_full_state_property(
+        >>> check_forward_full_state_property(  # doctest: +ELLIPSIS
         ...     ConfusionMatrix,
         ...     init_args = {'num_classes': 3},
         ...     input_args = {'preds': torch.randint(3, (10,)), 'target': torch.randint(3, (10,))},
-        ... )  # doctest: +ELLIPSIS
+        ... )
         Full state for 10 steps took: ...
         Partial state for 10 steps took: ...
         Full state for 100 steps took: ...