Python 3.9

.github/workflows/tests.yaml

@@ -9,11 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # MacOS tests are currently afflicted by a very high level of flakiness. On
-        # Linux and Windows, run on merge to master and on all pull requests.
-        # On MacOS, run only on merge to master (see twin workflow tests-macos.yaml).
-        os: [ubuntu-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        exclude:
+          # Several failures; Python 3.6 to be phased out soon
+          - os: macos-latest
+            python-version: "3.6"
 
         # Uncomment to stress-test the test suite for random failures
         # This will take a LONG time and delay all PRs across the whole github.com/dask!

continuous_integration/environment-3.8.yaml

@@ -2,7 +2,6 @@ name: dask-distributed
 channels:
   - conda-forge
   - defaults
-  - pytorch
 dependencies:
   - python=3.8
   - pip
@@ -11,13 +10,12 @@ dependencies:
   - click
   - cloudpickle
   - dask  # overridden by git tip below
-  - filesystem-spec  # overridden by git tip below
+  - filesystem-spec
   - h5py
   - ipykernel
   - ipywidgets
-  - joblib  # overridden by git tip below
+  - joblib
   - jupyter_client
-  - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
@@ -29,24 +27,16 @@ dependencies:
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
-  - python-blosc  # Only tested here
-  - python-snappy  # Only tested here
-  - pytorch  # Only tested here
   - requests
-  - s3fs  # overridden by git tip below
+  - s3fs
   - scikit-learn
   - scipy
   - sortedcollections
   - tblib
   - toolz
-  - torchvision  # Only tested here
   - tornado=6
-  - zict  # overridden by git tip below
+  - zict
   - zstandard
   - pip:
       - git+https://github.com/dask/dask
-      - git+https://github.com/dask/s3fs
-      - git+https://github.com/dask/zict
-      - git+https://github.com/intake/filesystem_spec
-      - git+https://github.com/joblib/joblib
       - keras

continuous_integration/environment-3.9.yaml

@@ -0,0 +1,52 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+  - pytorch
+dependencies:
+  - python=3.9
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - dask  # overridden by git tip below
+  - filesystem-spec  # overridden by git tip below
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib  # overridden by git tip below
+  - jupyter_client
+  - lz4  # Only tested here
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - python-blosc  # Only tested here
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
+  - requests
+  - s3fs  # overridden by git tip below
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - torchvision  # Only tested here
+  - tornado=6
+  - zict  # overridden by git tip below
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/s3fs
+      - git+https://github.com/dask/zict
+      - git+https://github.com/intake/filesystem_spec
+      - git+https://github.com/joblib/joblib
+      - keras

distributed/cli/tests/test_dask_worker.py

@@ -41,7 +41,7 @@ def test_nanny_worker_ports(loop):
                     if d["workers"]:
                         break
                     else:
-                        assert time() - start < 5
+                        assert time() - start < 60
                         sleep(0.1)
                 assert (
                     d["workers"]["tcp://127.0.0.1:9684"]["nanny"]

distributed/dashboard/tests/test_scheduler_bokeh.py

@@ -15,9 +15,10 @@
 import dask
 from dask.core import flatten
 from dask.utils import stringify
-from distributed.utils import format_dashboard_link
 from distributed.client import wait
+from distributed.compatibility import MACOS
 from distributed.metrics import time
+from distributed.utils import format_dashboard_link
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
 from distributed.dashboard.components.worker import Counters
 from distributed.dashboard.scheduler import applications
@@ -749,6 +750,7 @@ async def test_aggregate_action(c, s, a, b):
     assert ("compute") in mbk.action_source.data["names"]
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 async def test_compute_per_key(c, s, a, b):
     mbk = ComputePerKey(s)

distributed/deploy/tests/test_adaptive.py

@@ -302,7 +302,7 @@ async def test_adapt_down():
             start = time()
             while len(cluster.scheduler.workers) != 2:
                 await asyncio.sleep(0.1)
-                assert time() < start + 3
+                assert time() < start + 60
 
 
 @gen_test(timeout=30)

distributed/deploy/tests/test_spec_cluster.py

@@ -196,6 +196,7 @@ async def test_unexpected_closed_worker(cleanup):
             assert len(cluster.workers) == 2
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_restart(cleanup):

distributed/deploy/tests/test_ssh.py

@@ -81,7 +81,7 @@ def test_defer_to_old(loop):
 
 
 @pytest.mark.avoid_ci
-def test_old_ssh_wih_local_dir(loop):
+def test_old_ssh_with_local_dir(loop):
     with pytest.warns(Warning):
         from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
 

distributed/utils.py

@@ -64,6 +64,12 @@
 except ImportError:
     thread_state = threading.local()
 
+# For some reason this is required in python >= 3.9
+if WINDOWS:
+    import multiprocessing.popen_spawn_win32
+else:
+    import multiprocessing.popen_spawn_posix
+
 logger = _logger = logging.getLogger(__name__)
 
 

setup.py

@@ -92,6 +92,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
     ],