Python 3.9

.github/workflows/tests-macos.yaml

@@ -4,7 +4,7 @@ on:
   - push
   # MacOS tests are currently afflicted by a very high level of flakiness; do not run on
   # PR unless this line is uncommented.
-  # - pull_request
+  - pull_request
 
 jobs:
   test:
@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         # Many deterministic test failures on Python 3.6
-        python-version: ["3.7", "3.8"]
+        python-version: ["3.7", "3.8", "3.9"]
 
         # Uncomment to stress-test the test suite for random failures
         # This will take a LONG time and delay all PRs across the whole github.com/dask!

.github/workflows/tests.yaml

@@ -13,7 +13,7 @@ jobs:
         # Linux and Windows, run on merge to master and on all pull requests.
         # On MacOS, run only on merge to master (see twin workflow tests-macos.yaml).
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8"]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
 
         # Uncomment to stress-test the test suite for random failures
         # This will take a LONG time and delay all PRs across the whole github.com/dask!

continuous_integration/environment-3.8.yaml

@@ -2,7 +2,6 @@ name: dask-distributed
 channels:
   - conda-forge
   - defaults
-  - pytorch
 dependencies:
   - python=3.8
   - pip
@@ -11,13 +10,12 @@ dependencies:
   - click
   - cloudpickle
   - dask  # overridden by git tip below
-  - filesystem-spec  # overridden by git tip below
+  - filesystem-spec
   - h5py
   - ipykernel
   - ipywidgets
-  - joblib  # overridden by git tip below
+  - joblib
   - jupyter_client
-  - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
@@ -29,24 +27,16 @@ dependencies:
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
-  - python-blosc  # Only tested here
-  - python-snappy  # Only tested here
-  - pytorch  # Only tested here
   - requests
-  - s3fs  # overridden by git tip below
+  - s3fs
   - scikit-learn
   - scipy
   - sortedcollections
   - tblib
   - toolz
-  - torchvision  # Only tested here
   - tornado=6
-  - zict  # overridden by git tip below
+  - zict
   - zstandard
   - pip:
       - git+https://github.com/dask/dask
-      - git+https://github.com/dask/s3fs
-      - git+https://github.com/dask/zict
-      - git+https://github.com/intake/filesystem_spec
-      - git+https://github.com/joblib/joblib
       - keras

continuous_integration/environment-3.9.yaml

@@ -0,0 +1,52 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+  - pytorch
+dependencies:
+  - python=3.9
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - dask  # overridden by git tip below
+  - filesystem-spec  # overridden by git tip below
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib  # overridden by git tip below
+  - jupyter_client
+  - lz4  # Only tested here
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - python-blosc  # Only tested here
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
+  - requests
+  - s3fs  # overridden by git tip below
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - torchvision  # Only tested here
+  - tornado=6
+  - zict  # overridden by git tip below
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/s3fs
+      - git+https://github.com/dask/zict
+      - git+https://github.com/intake/filesystem_spec
+      - git+https://github.com/joblib/joblib
+      - keras

distributed/cli/tests/test_dask_worker.py

@@ -41,7 +41,7 @@ def test_nanny_worker_ports(loop):
                     if d["workers"]:
                         break
                     else:
-                        assert time() - start < 5
+                        assert time() - start < 60
                         sleep(0.1)
                 assert (
                     d["workers"]["tcp://127.0.0.1:9684"]["nanny"]

distributed/deploy/tests/test_ssh.py

@@ -81,7 +81,7 @@ def test_defer_to_old(loop):
 
 
 @pytest.mark.avoid_ci
-def test_old_ssh_wih_local_dir(loop):
+def test_old_ssh_with_local_dir(loop):
     with pytest.warns(Warning):
         from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
 

distributed/utils.py

@@ -10,7 +10,6 @@
 import html
 import json
 import logging
-import multiprocessing
 import os
 import re
 import shutil
@@ -71,6 +70,12 @@
 
 
 def _initialize_mp_context():
+    import multiprocessing  # noqa: F401
+
+    if not WINDOWS:
+        # For some reason this is required in python >= 3.9
+        import multiprocessing.popen_spawn_posix
+
     if WINDOWS or PYPY:
         return multiprocessing
     else:

setup.py

@@ -92,6 +92,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
     ],