[tests] rerun failed tests 2 times in CI

[timkpaine]

References

Code changes

Adds a new test dependency and updates CI tests to rerun failures 2 times to minimize the chance of flaky tests failing.

User-facing changes

None

Backwards-incompatible changes

N/A

[github-actions]

👈 Launch a Binder on branch timkpaine/voila/tkp/rerunfailures

[github-actions]

Benchmark report

The execution time (in milliseconds) are grouped by test file, test type and browser.
For each case, the following values are computed: min <- [1st quartile - median - 3rd quartile] -> max.

Results table

Test file	voila-tree-classic.ipynb	voila-tree-light.ipynb	voila-tree-dark.ipynb	voila-tree-miami.ipynb	basics.ipynb	bqplot.ipynb	dashboard.ipynb	gridspecLayout.ipynb	interactive.ipynb	ipympl.ipynb	ipyvolume.ipynb	multiple_widgets.ipynb	query-strings.ipynb	reveal.ipynb
Render
chromium
actual	91 <- [99 - 117 - 145] -> 234	73 <- [76 - 83 - 100] -> 136	68 <- [71 - 80 - 97] -> 123	70 <- [74 - 81 - 95] -> 133	2486 <- [2625 - 2743 - 2810] -> 3383	2373 <- [2392 - 2414 - 2478] -> 2713	2711 <- [2716 - 2781 - 2791] -> 2918	2609 <- [2613 - 2689 - 2803] -> 3019	1876 <- [1901 - 1905 - 1932] -> 2062	3427 <- [3486 - 3602 - 3787] -> 4350	7378 <- [8601 - 8745 - 8935] -> 9054	6285 <- [6315 - 6329 - 6356] -> 6407	1238 <- [1394 - 1408 - 1423] -> 1445	2386 <- [2392 - 2408 - 2505] -> 2752
expected	3379 <- [3442 - 3517 - 3701] -> 3876	2976 <- [3227 - 3321 - 3421] -> 3604	3608 <- [3623 - 3709 - 3793] -> 3825	4453 <- [4453 - 4523 - 4661] -> 4748	2559 <- [2655 - 2656 - 2660] -> 2674	3982 <- [4079 - 4213 - 4356] -> 4743	12183 <- [18509 - 19553 - 20811] -> 21515	15319 <- [15660 - 15796 - 15912] -> 16056	1517 <- [1920 - 1997 - 2103] -> 2113

❗ Test metadata have changed

--- /dev/fd/63	2022-04-29 01:34:50.671198134 +0000
+++ /dev/fd/62	2022-04-29 01:34:50.671198134 +0000
@@ -4,37 +4,37 @@
     "BENCHMARK_REFERENCE": "actual"
   },
   "browsers": {
-    "chromium": "97.0.4666.0"
+    "chromium": "94.0.4595.0"
   },
   "systemInformation": {
     "cpu": {
-      "brand": "Xeon® Platinum 8272CL",
+      "brand": "Xeon® E5-2673 v3",
       "cache": {
         "l1d": 65536,
         "l1i": 65536,
-        "l2": 2097152,
-        "l3": 36700160
+        "l2": 524288,
+        "l3": 31457280
       },
       "cores": 2,
       "family": "6",
-      "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt avx512cd avx512bw avx512vl xsaveopt xsavec xsaves md_clear",
+      "flags": "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm invpcid_single pti fsgsbase bmi1 avx2 smep bmi2 erms invpcid xsaveopt md_clear",
       "governor": "",
       "manufacturer": "Intel®",
-      "model": "85",
+      "model": "63",
       "physicalCores": 2,
       "processors": 1,
       "revision": "",
       "socket": "",
-      "speed": 2.6,
+      "speed": 2.4,
       "speedMax": null,
       "speedMin": null,
-      "stepping": "7",
+      "stepping": "2",
       "vendor": "GenuineIntel",
       "virtualization": false,
       "voltage": ""
     },
     "mem": {
-      "total": 7283843072
+      "total": 7291699200
     },
     "osInfo": {
       "arch": "x64",
@@ -42,11 +42,11 @@
       "codename": "Focal Fossa",
       "codepage": "UTF-8",
       "distro": "Ubuntu",
-      "kernel": "5.13.0-1022-azure",
+      "kernel": "5.8.0-1040-azure",
       "logofile": "ubuntu",
       "platform": "linux",
-      "release": "20.04.4 LTS",
-      "serial": "d7e36e900f074855bb7a05a49f71c059",
+      "release": "20.04.3 LTS",
+      "serial": "cfc067bfcb844f35865e279a1b0e66c5",
       "servicepack": "",
       "uefi": false
     }

[trungleduc]

Thanks!