From 3a7fe8714a8beb9b7e99718be1492ba336f829a9 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 23 Dec 2020 13:34:23 -0600 Subject: [PATCH 01/12] Drop 3.6 builds in CI --- .github/workflows/ci-windows.yaml | 9 +-------- .travis.yml | 3 +-- continuous_integration/travis/install.sh | 4 ---- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml index 6d5923eb187..4c210ef6f06 100644 --- a/.github/workflows/ci-windows.yaml +++ b/.github/workflows/ci-windows.yaml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8"] + python-version: ["3.7", "3.8"] steps: - name: Checkout source @@ -23,13 +23,6 @@ jobs: activate-environment: dask-distributed auto-activate-base: false - - name: Install contextvars - shell: bash -l {0} - run: | - if [[ "${{ matrix.python-version }}" = "3.6" ]]; then - conda install -c conda-forge contextvars - fi - - name: Install tornado shell: bash -l {0} run: | diff --git a/.travis.yml b/.travis.yml index 850fcf23d53..307d1f17e87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,7 @@ dist: trusty env: matrix: - - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="lz4" TORNADO=5 CRICK=true - - PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6 + - PYTHON=3.7 TESTS=true COVERAGE=true PACKAGES="lz4 python-snappy python-blosc" CRICK=true TORNADO=5 - PYTHON=3.8 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6 matrix: diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh index d7a54093ce0..6e0fc2f799e 100644 --- a/continuous_integration/travis/install.sh +++ b/continuous_integration/travis/install.sh @@ -67,10 +67,6 @@ conda create -n dask-distributed -c conda-forge -c defaults \ source activate dask-distributed -if [[ $PYTHON == 3.6 ]]; then - conda install -c conda-forge -c defaults contextvars -fi - if [[ $PYTHON == 3.8 ]]; then # Install pytorch to run related tests conda install -c pytorch -c conda-forge -c defaults pytorch torchvision From 598b2ff28d86a9f80b9e400626545386d849c25c Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 23 Dec 2020 13:34:43 -0600 Subject: [PATCH 02/12] Update dependencies --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91a18f6346b..92f58470cbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ click >= 6.6 cloudpickle >= 1.5.0 -contextvars;python_version<'3.7' dask>=2020.12.0 msgpack >= 0.6.0 psutil >= 5.0 From e3b47a20766fc509039d32397000bc0c41f3a4b5 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 23 Dec 2020 13:35:00 -0600 Subject: [PATCH 03/12] Update setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9160e1ffdc9..dbd62536f30 100755 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ url="https://distributed.dask.org", maintainer="Matthew Rocklin", maintainer_email="mrocklin@gmail.com", - python_requires=">=3.6", + python_requires=">=3.7", license="BSD", package_data={ "": ["templates/index.html", "template.html"], @@ -89,7 +89,6 @@ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Topic :: Scientific/Engineering", From 04825c182b3ee2c16fbe9b036faef75f673a81e1 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 23 Dec 2020 13:36:47 -0600 Subject: [PATCH 04/12] Remove compatibility code --- distributed/cli/tests/test_dask_scheduler.py | 3 --- distributed/client.py | 9 --------- distributed/comm/tests/test_comms.py | 3 +-- distributed/compatibility.py | 13 ------------- distributed/deploy/tests/test_local.py | 3 --- distributed/pubsub.py | 4 +--- distributed/tests/test_client.py | 5 ----- distributed/tests/test_core.py | 5 ----- distributed/tests/test_diskutils.py | 2 -- distributed/tests/test_scheduler.py | 3 --- distributed/utils.py | 4 ++-- distributed/variable.py | 3 +-- 12 files changed, 5 insertions(+), 52 deletions(-) diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py index 470bfddf81d..5652a685fa8 100644 --- a/distributed/cli/tests/test_dask_scheduler.py +++ b/distributed/cli/tests/test_dask_scheduler.py @@ -185,9 +185,6 @@ def test_interface(loop): assert all("127.0.0.1" == d["host"] for d in info["workers"].values()) -@pytest.mark.skipif( - sys.version_info < (3, 7), reason="Intermittent failure on old Python version" -) def test_pid_file(loop): def check_pidfile(proc, pidfile): start = time() diff --git a/distributed/client.py b/distributed/client.py index a7887ef4b9d..a91d58b592f 100644 --- a/distributed/client.py +++ b/distributed/client.py @@ -758,15 +758,6 @@ def as_current(self): method to return self. Any Future objects deserialized inside this context manager will be automatically attached to this Client. """ - # In Python 3.6, contextvars are thread-local but not Task-local. - # We can still detect a race condition though. - if sys.version_info < (3, 7) and _current_client.get() not in (self, None): - raise RuntimeError( - "Detected race condition where multiple asynchronous clients tried " - "entering the as_current() context manager at the same time. " - "Please upgrade to Python 3.7+." - ) - tok = _current_client.set(self) try: yield diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py index e94767de2df..0173463823a 100644 --- a/distributed/comm/tests/test_comms.py +++ b/distributed/comm/tests/test_comms.py @@ -222,8 +222,7 @@ async def handle_comm(comm): assert not handle_comm_called writer.close() - if hasattr(writer, "wait_closed"): # always true for python >= 3.7, but not for 3.6 - await writer.wait_closed() + await writer.wait_closed() @pytest.mark.asyncio diff --git a/distributed/compatibility.py b/distributed/compatibility.py index fc64b7568b5..97abf631de8 100644 --- a/distributed/compatibility.py +++ b/distributed/compatibility.py @@ -10,16 +10,3 @@ PYPY = platform.python_implementation().lower() == "pypy" WINDOWS = sys.platform.startswith("win") TORNADO6 = tornado.version_info[0] >= 6 -PY37 = sys.version_info[:2] >= (3, 7) - -if sys.version_info[:2] >= (3, 7): - from asyncio import get_running_loop -else: - - def get_running_loop(): - from asyncio import _get_running_loop - - loop = _get_running_loop() - if loop is None: - raise RuntimeError("no running event loop") - return loop diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py index 2b63f28b10f..51fb4054c94 100644 --- a/distributed/deploy/tests/test_local.py +++ b/distributed/deploy/tests/test_local.py @@ -1031,9 +1031,6 @@ async def test_capture_security(cleanup, temporary): @pytest.mark.asyncio -@pytest.mark.skipif( - sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented" -) async def test_no_danglng_asyncio_tasks(cleanup): start = asyncio.all_tasks() async with LocalCluster(asynchronous=True, processes=False): diff --git a/distributed/pubsub.py b/distributed/pubsub.py index 3aeec084df1..500f8bbaffe 100644 --- a/distributed/pubsub.py +++ b/distributed/pubsub.py @@ -1,6 +1,5 @@ import asyncio from collections import defaultdict, deque -from contextlib import suppress import logging import threading import weakref @@ -421,8 +420,7 @@ async def _(): try: await asyncio.wait_for(_(), timeout2) finally: - with suppress(RuntimeError): # Python 3.6 fails here sometimes - self.condition.release() + self.condition.release() return self.buffer.popleft() diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py index 0625bf8a0fa..8c9fad44afa 100644 --- a/distributed/tests/test_client.py +++ b/distributed/tests/test_client.py @@ -3670,7 +3670,6 @@ async def test_reconnect_timeout(c, s): @pytest.mark.slow @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows") -@pytest.mark.skipif(sys.version_info < (3, 7), reason="TODO: intermittent failures") @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)]) def test_open_close_many_workers(loop, worker, count, repeat): psutil = pytest.importorskip("psutil") @@ -4055,10 +4054,6 @@ def run2(): t2.join() -@pytest.mark.xfail( - sys.version_info < (3, 7), - reason="Python 3.6 contextvars are not copied on Task creation", -) @gen_cluster(client=False) async def test_as_current_is_task_local(s, a, b): l1 = asyncio.Lock() diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py index b43b143721f..abd5de27041 100644 --- a/distributed/tests/test_core.py +++ b/distributed/tests/test_core.py @@ -1,7 +1,6 @@ import asyncio import os import socket -import sys import threading import weakref import warnings @@ -209,10 +208,6 @@ class MyServer(Server): default_port = 8756 -@pytest.mark.skipif( - sys.version_info < (3, 7), - reason="asynccontextmanager not avaiable before Python 3.7", -) @pytest.mark.asyncio async def test_server_listen(): """ diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py index f69485cfa46..a2ac01f8b12 100644 --- a/distributed/tests/test_diskutils.py +++ b/distributed/tests/test_diskutils.py @@ -276,8 +276,6 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs): def test_workspace_concurrency(tmpdir): if WINDOWS: raise pytest.xfail.Exception("TODO: unknown failure on windows") - if sys.version_info < (3, 7): - raise pytest.xfail.Exception("TODO: unknown failure on Python 3.6") _test_workspace_concurrency(tmpdir, 5.0, 6) diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py index ec44ca85560..80d5a278f6e 100644 --- a/distributed/tests/test_scheduler.py +++ b/distributed/tests/test_scheduler.py @@ -1811,9 +1811,6 @@ async def test_get_task_duration(c, s, a, b): @pytest.mark.asyncio -@pytest.mark.skipif( - sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented" -) async def test_no_danglng_asyncio_tasks(cleanup): start = asyncio.all_tasks() async with Scheduler(port=0) as s: diff --git a/distributed/utils.py b/distributed/utils.py index 30044740f64..30b2d43b3b3 100644 --- a/distributed/utils.py +++ b/distributed/utils.py @@ -55,7 +55,7 @@ except ImportError: PollIOLoop = None # dropped in tornado 6.0 -from .compatibility import PYPY, WINDOWS, get_running_loop +from .compatibility import PYPY, WINDOWS from .metrics import time @@ -1156,7 +1156,7 @@ def reset_logger_locks(): if is_kernel(): try: - get_running_loop() + asyncio.get_running_loop() except RuntimeError: is_kernel_and_no_running_loop = True diff --git a/distributed/variable.py b/distributed/variable.py index db8da76e44c..bf4cbef803a 100644 --- a/distributed/variable.py +++ b/distributed/variable.py @@ -89,8 +89,7 @@ async def _(): # Python 3.6 is odd and requires special help here await asyncio.wait_for(_(), timeout=left) finally: - with suppress(RuntimeError): # Python 3.6 loses lock on finally clause - self.started.release() + self.started.release() record = self.variables[name] if record["type"] == "Future": From 4dce37d38b0c510271cce5e76eb7cbe0411544d3 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 23 Dec 2020 13:37:17 -0600 Subject: [PATCH 05/12] Update note in installation docs --- docs/source/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/install.rst b/docs/source/install.rst index db1bf316400..695cd3e777f 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -38,6 +38,6 @@ Notes `_. with Python from macports that makes executables be placed in a location that is not available by default. A simple solution is to extend the ``PATH`` environment variable to the location -where Python from macports install the binaries. For example, for Python 3.6:: +where Python from macports install the binaries. For example, for Python 3.7:: - $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.6/bin:$PATH + $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.7/bin:$PATH From 915d8995390e878139fdd8dd784bdeb2bbf779b6 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 4 Mar 2021 17:11:24 -0600 Subject: [PATCH 06/12] Remove Python 3.6 build from CI --- .github/workflows/tests.yaml | 6 +-- continuous_integration/environment-3.6.yaml | 47 --------------------- 2 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 continuous_integration/environment-3.6.yaml diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c27ae784ea8..d9baee97c46 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -10,11 +10,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ["3.6", "3.7", "3.8", "3.9"] - exclude: - # Several failures; Python 3.6 to be phased out soon - - os: macos-latest - python-version: "3.6" + python-version: ["3.7", "3.8", "3.9"] # Uncomment to stress-test the test suite for random failures # This will take a LONG time and delay all PRs across the whole github.com/dask! diff --git a/continuous_integration/environment-3.6.yaml b/continuous_integration/environment-3.6.yaml deleted file mode 100644 index 7a13f63b232..00000000000 --- a/continuous_integration/environment-3.6.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: dask-distributed -channels: - - conda-forge - - defaults -dependencies: - - python=3.6 - - pip - - asyncssh - - bokeh - - click - - cloudpickle - - contextvars # Backport from Python 3.7 - - dask # overridden by git tip below - - filesystem-spec - - h5py - - ipykernel - - ipywidgets - - joblib - - jupyter_client - - msgpack-python - - netcdf4 - - paramiko - - pickle5 # Backport from Python 3.8 - - prometheus_client - - psutil - - pytest - - pytest-asyncio<0.14.0 - - pytest-faulthandler - - pytest-repeat - - pytest-rerunfailures - - pytest-timeout - - requests - - s3fs - - scikit-learn - - scipy - - sortedcollections - - tblib - - toolz - - tornado=5 # Only tested here - - zict - - zstandard - - pip: - # The '.git@master' is a hack needed to work around the failure - # ERROR: Package 'dask' requires a different Python: 3.6.12 not in '>=3.6' - # It only happens in CI; cannot reproduce locally - - git+https://github.com/dask/dask.git@master - - keras From efe4c40bccdb8cd4d144768eb615e41ce8702165 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 4 Mar 2021 17:12:32 -0600 Subject: [PATCH 07/12] Make sure a CI build has tornado=5 --- continuous_integration/environment-3.7.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml index 7057155b470..bb7838fe205 100644 --- a/continuous_integration/environment-3.7.yaml +++ b/continuous_integration/environment-3.7.yaml @@ -36,7 +36,7 @@ dependencies: - sortedcollections - tblib - toolz - - tornado=6 + - tornado=5 # Only tested here - zict - zstandard - pip: From 05452f17567d74f37ab1fa0df31821332cfeccd8 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 8 Mar 2021 15:08:01 -0600 Subject: [PATCH 08/12] Temporarily point to Python 3.7 CI build to Dask PR #7006 --- continuous_integration/environment-3.7.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml index bb7838fe205..94563620ce5 100644 --- a/continuous_integration/environment-3.7.yaml +++ b/continuous_integration/environment-3.7.yaml @@ -40,6 +40,7 @@ dependencies: - zict - zstandard - pip: - - git+https://github.com/dask/dask + # TODO: Point back to dask/dask before merging + - git+https://github.com/jrbourbeau/dask.git@drop-py3.6 - git+https://github.com/jcrist/crick # Only tested here - keras From 41fc9a24ffc3cd416200f2908ffc81725459a2b8 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 8 Mar 2021 15:35:37 -0600 Subject: [PATCH 09/12] Update timing functions --- distributed/metrics.py | 67 +++++++----------------------------------- 1 file changed, 10 insertions(+), 57 deletions(-) diff --git a/distributed/metrics.py b/distributed/metrics.py index 0f7d78a8129..163a982b792 100755 --- a/distributed/metrics.py +++ b/distributed/metrics.py @@ -84,60 +84,13 @@ def resync(self): # Under modern Unices, time.time() should be good enough time = timemod.time - -def _native_thread_time(): - # Python 3.7+, not all platforms - return timemod.thread_time() - - -def _linux_thread_time(): - # Use hardcoded CLOCK_THREAD_CPUTIME_ID on Python 3 <= 3.6 - if sys.platform != "linux": - raise OSError - return timemod.clock_gettime(3) - - -def _native_process_time(): - # Python 3, should work everywhere - return timemod.process_time() - - -def _native_clock_func(): - # time.clock() unfortunately has different semantics depending on the - # platform. On POSIX it's a per-process CPU timer (with possibly - # poor resolution). On Windows it's a high-resolution wall clock timer. - return timemod.clock() - - -def _detect_process_time(): - """ - Return a per-process CPU timer function if possible, otherwise - a wall-clock timer. - """ - for func in [_native_process_time]: - try: - func() - return func - except (AttributeError, OSError): - pass - # Only Python 2? - return _native_clock_func - - -def _detect_thread_time(): - """ - Return a per-thread CPU timer function if possible, otherwise - a per-process CPU timer function, or at worse a wall-clock timer. - """ - for func in [_native_thread_time, _linux_thread_time, _native_process_time]: - try: - func() - return func - except (AttributeError, OSError): - pass - # Only Python 2? - return time - - -process_time = _detect_process_time() -thread_time = _detect_thread_time() +process_time = timemod.process_time + +# Get a per-thread CPU timer function if possible, otherwise +# use a per-process CPU timer function. +try: + # thread_time is supported on Python 3.7+ but not all platforms + thread_time = timemod.thread_time +except (AttributeError, OSError): + # process_time is supported on Python 3.3+ everywhere + thread_time = process_time From 1f6b25deb68f3d327558b732b37b4de02953895f Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 8 Mar 2021 17:11:26 -0600 Subject: [PATCH 10/12] Make sure tags are fetched --- .github/workflows/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d9baee97c46..9d6b6a526c4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -19,6 +19,8 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Setup Conda Environment uses: conda-incubator/setup-miniconda@v2 From 3793d5de61ff30665209be31eefb956042b5d054 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 8 Mar 2021 22:07:51 -0600 Subject: [PATCH 11/12] Trigger CI after pushing tags to jrbourbeau/dask From 56f5adc8adbcbf141e5266fffc633d020d48ea01 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Tue, 9 Mar 2021 09:52:35 -0600 Subject: [PATCH 12/12] Point CI back to dask/dask --- continuous_integration/environment-3.7.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml index 94563620ce5..bb7838fe205 100644 --- a/continuous_integration/environment-3.7.yaml +++ b/continuous_integration/environment-3.7.yaml @@ -40,7 +40,6 @@ dependencies: - zict - zstandard - pip: - # TODO: Point back to dask/dask before merging - - git+https://github.com/jrbourbeau/dask.git@drop-py3.6 + - git+https://github.com/dask/dask - git+https://github.com/jcrist/crick # Only tested here - keras