diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 97957f15a..af7a0d6d3 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -12,9 +12,11 @@ concurrency: jobs: tests: runs-on: ubuntu-latest + timeout-minutes: 20 strategy: matrix: python-version: ["3.9"] + fail-fast: false steps: - name: Checkout uses: actions/checkout@v2 @@ -50,7 +52,15 @@ jobs: with: package_name: jupyter_server - # Test using jupyter_kernel_test + jupyter_kernel_test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Base Setup + uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - name: Setup conda ${{ matrix.python-version }} uses: conda-incubator/setup-miniconda@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50d01c41e..12eae70ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,8 +45,8 @@ jobs: build-n-test-n-coverage: name: Build, test and code coverage - runs-on: ${{ matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false @@ -94,6 +94,7 @@ jobs: docs: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v2 @@ -111,6 +112,7 @@ jobs: test_miniumum_verisons: name: Test Minimum Versions runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v2 - name: Base Setup @@ -124,6 +126,7 @@ jobs: test_prereleases: name: Test Prereleases + timeout-minutes: 10 runs-on: ubuntu-latest steps: - name: Checkout @@ -144,7 +147,7 @@ jobs: make_sdist: name: Make SDist runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 10 steps: - uses: actions/checkout@v2 - name: Base Setup diff --git a/jupyter_client/client.py b/jupyter_client/client.py index d71eac5ab..4e5e7e0d4 100644 --- a/jupyter_client/client.py +++ b/jupyter_client/client.py @@ -11,6 +11,7 @@ import zmq.asyncio from traitlets import Any # type: ignore +from traitlets import Bool from traitlets import Instance from traitlets import Type @@ -92,7 +93,10 @@ class KernelClient(ConnectionFileMixin): # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.asyncio.Context) + _created_context: Bool = Bool(False) + def _context_default(self) -> zmq.asyncio.Context: + self._created_context = True return zmq.asyncio.Context() # The classes to use for the various channels @@ -282,6 +286,9 @@ def start_channels( :meth:`start_kernel`. If the channels have been stopped and you call this, :class:`RuntimeError` will be raised. """ + # Create the context if needed. + if not self._created_context: + self.context = self._context_default() if iopub: self.iopub_channel.start() if shell: @@ -311,6 +318,9 @@ def stop_channels(self) -> None: self.hb_channel.stop() if self.control_channel.is_alive(): self.control_channel.stop() + if self._created_context: + self._created_context = False + self.context.destroy() @property def channels_running(self) -> bool: diff --git a/jupyter_client/kernelspec.py b/jupyter_client/kernelspec.py index 9252c8243..8d83842ea 100644 --- a/jupyter_client/kernelspec.py +++ b/jupyter_client/kernelspec.py @@ -31,6 +31,8 @@ class KernelSpec(HasTraits): argv = List() + name = Unicode() + mimetype = Unicode() display_name = Unicode() language = Unicode() env = Dict() diff --git a/jupyter_client/kernelspecapp.py b/jupyter_client/kernelspecapp.py index 0690840e2..32f6acd97 100644 --- a/jupyter_client/kernelspecapp.py +++ b/jupyter_client/kernelspecapp.py @@ -182,7 +182,7 @@ def _kernel_spec_manager_default(self): return KernelSpecManager(data_dir=self.data_dir, parent=self) flags = { - "f": ({"RemoveKernelSpec": {"force": True}}, force.get_metadata("help")), + "f": ({"RemoveKernelSpec": {"force": True}}, force.help), } flags.update(JupyterApp.flags) diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index bc3190f25..969985ab9 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -92,6 +92,7 @@ def __init__(self, *args, **kwargs): self._shutdown_status = _ShutdownStatus.Unset # Create a place holder future. try: + asyncio.get_running_loop() self._ready = Future() except RuntimeError: # No event loop running, use concurrent future @@ -476,7 +477,8 @@ async def _async_shutdown_kernel(self, now: bool = False, restart: bool = False) # Stop monitoring for restarting while we shutdown. self.stop_restarter() - await ensure_async(self.interrupt_kernel()) + if self.has_kernel: + await ensure_async(self.interrupt_kernel()) if now: await ensure_async(self._kill_kernel()) diff --git a/jupyter_client/provisioning/local_provisioner.py b/jupyter_client/provisioning/local_provisioner.py index 043068680..08cb9aa14 100644 --- a/jupyter_client/provisioning/local_provisioner.py +++ b/jupyter_client/provisioning/local_provisioner.py @@ -60,6 +60,11 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() + # Make sure all the fds get closed. + for attr in ['stdout', 'stderr', 'stdin']: + fid = getattr(self.process, attr) + if fid: + fid.close() self.process = None # allow has_process to now return False return ret diff --git a/jupyter_client/ssh/tunnel.py b/jupyter_client/ssh/tunnel.py index 88e10323f..40826c736 100644 --- a/jupyter_client/ssh/tunnel.py +++ b/jupyter_client/ssh/tunnel.py @@ -36,8 +36,6 @@ class SSHException(Exception): # type: ignore except ImportError: pexpect = None -from zmq.utils.strtypes import b - def select_random_ports(n): """Select and return n random ports that are available.""" @@ -56,7 +54,7 @@ def select_random_ports(n): # ----------------------------------------------------------------------------- # Check for passwordless login # ----------------------------------------------------------------------------- -_password_pat = re.compile(b(r"pass(word|phrase):"), re.IGNORECASE) +_password_pat = re.compile((r"pass(word|phrase):".encode("utf8")), re.IGNORECASE) def try_passwordless_ssh(server, keyfile, paramiko=None): diff --git a/jupyter_client/tests/conftest.py b/jupyter_client/tests/conftest.py index 8f9ad7378..b52872a89 100644 --- a/jupyter_client/tests/conftest.py +++ b/jupyter_client/tests/conftest.py @@ -7,9 +7,30 @@ from .utils import test_env +try: + import resource +except ImportError: + # Windows + resource = None + pjoin = os.path.join +# Handle resource limit +# Ensure a minimal soft limit of DEFAULT_SOFT if the current hard limit is at least that much. +if resource is not None: + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + + DEFAULT_SOFT = 4096 + if hard >= DEFAULT_SOFT: + soft = DEFAULT_SOFT + + if hard < soft: + hard = soft + + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + + if os.name == "nt" and sys.version_info >= (3, 7): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) diff --git a/jupyter_client/tests/test_client.py b/jupyter_client/tests/test_client.py index a422462b3..867749f68 100644 --- a/jupyter_client/tests/test_client.py +++ b/jupyter_client/tests/test_client.py @@ -28,8 +28,12 @@ def setUp(self): except NoSuchKernel: pytest.skip() self.km, self.kc = start_new_kernel(kernel_name=NATIVE_KERNEL_NAME) - self.addCleanup(self.kc.stop_channels) - self.addCleanup(self.km.shutdown_kernel) + + def tearDown(self): + self.env_patch.stop() + self.km.shutdown_kernel() + self.kc.stop_channels() + return super().tearDown() def test_execute_interactive(self): kc = self.kc diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index 983ca8095..13a6f5422 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -440,6 +440,7 @@ def execute(cmd): km.shutdown_kernel() assert km.context.closed + kc.stop_channels() @pytest.mark.asyncio diff --git a/jupyter_client/tests/test_multikernelmanager.py b/jupyter_client/tests/test_multikernelmanager.py index 8cd953d60..bf24496e0 100644 --- a/jupyter_client/tests/test_multikernelmanager.py +++ b/jupyter_client/tests/test_multikernelmanager.py @@ -44,6 +44,10 @@ def setUp(self): self.env_patch.start() super().setUp() + def tearDown(self) -> None: + self.env_patch.stop() + return super().tearDown() + # static so picklable for multiprocessing on Windows @staticmethod def _get_tcp_km(): @@ -243,6 +247,10 @@ def setUp(self): self.env_patch.start() super().setUp() + def tearDown(self) -> None: + self.env_patch.stop() + return super().tearDown() + # static so picklable for multiprocessing on Windows @staticmethod def _get_tcp_km(): @@ -465,8 +473,9 @@ async def test_start_sequence_ipc_kernels(self): def tcp_lifecycle_with_loop(self): # Ensure each thread has an event loop - asyncio.set_event_loop(asyncio.new_event_loop()) - asyncio.get_event_loop().run_until_complete(self.raw_tcp_lifecycle()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self.raw_tcp_lifecycle()) # static so picklable for multiprocessing on Windows @classmethod @@ -479,11 +488,8 @@ async def raw_tcp_lifecycle(cls, test_kid=None): # static so picklable for multiprocessing on Windows @classmethod def raw_tcp_lifecycle_sync(cls, test_kid=None): - loop = asyncio.get_event_loop() - if loop.is_running(): - # Forked MP, make new loop - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) loop.run_until_complete(cls.raw_tcp_lifecycle(test_kid=test_kid)) @gen_test diff --git a/jupyter_client/tests/test_provisioning.py b/jupyter_client/tests/test_provisioning.py index f8063f272..de15ea011 100644 --- a/jupyter_client/tests/test_provisioning.py +++ b/jupyter_client/tests/test_provisioning.py @@ -66,6 +66,11 @@ async def wait(self) -> Optional[int]: # Process is no longer alive, wait and clear ret = self.process.wait() + # Make sure all the fds get closed. + for attr in ['stdout', 'stderr', 'stdin']: + fid = getattr(self.process, attr) + if fid: + fid.close() self.process = None return ret diff --git a/jupyter_client/tests/test_session.py b/jupyter_client/tests/test_session.py index 179c61f0e..bd5956143 100644 --- a/jupyter_client/tests/test_session.py +++ b/jupyter_client/tests/test_session.py @@ -10,6 +10,7 @@ import pytest import zmq +from tornado import ioloop from zmq.eventloop.zmqstream import ZMQStream from zmq.tests import BaseZMQTestCase @@ -171,7 +172,8 @@ def test_tracking(self): a, b = self.create_bound_pair(zmq.PAIR, zmq.PAIR) s = self.session s.copy_threshold = 1 - ZMQStream(a) + loop = ioloop.IOLoop(make_current=False) + ZMQStream(a, io_loop=loop) msg = s.send(a, "hello", track=False) self.assertTrue(msg["tracker"] is ss.DONE) msg = s.send(a, "hello", track=True) diff --git a/jupyter_client/tests/test_utils.py b/jupyter_client/tests/test_utils.py deleted file mode 100644 index dd6849192..000000000 --- a/jupyter_client/tests/test_utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -from unittest import mock - -import pytest - -from jupyter_client.utils import run_sync - - -@pytest.fixture -def loop(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop - - -def test_run_sync_clean_up_task(loop): - async def coro_never_called(): - pytest.fail("The call to this coroutine is not expected") - - # Ensure that run_sync cancels the pending task - with mock.patch.object(loop, "run_until_complete") as patched_loop: - patched_loop.side_effect = KeyboardInterrupt - with mock.patch("asyncio.ensure_future") as patched_ensure_future: - mock_future = mock.Mock() - patched_ensure_future.return_value = mock_future - with pytest.raises(KeyboardInterrupt): - run_sync(coro_never_called)() - mock_future.cancel.assert_called_once() - # Suppress 'coroutine ... was never awaited' warning - patched_ensure_future.call_args[0][0].close() diff --git a/jupyter_client/tests/utils.py b/jupyter_client/tests/utils.py index 5ca313469..ffaba2e9a 100644 --- a/jupyter_client/tests/utils.py +++ b/jupyter_client/tests/utils.py @@ -62,7 +62,11 @@ def start(self): def stop(self): self.env_patch.stop() - self.test_dir.cleanup() + try: + self.test_dir.cleanup() + except (PermissionError, NotADirectoryError): + if os.name != 'nt': + raise def __enter__(self): self.start() diff --git a/jupyter_client/utils.py b/jupyter_client/utils.py index f2f3c4dc4..9dea2bc2e 100644 --- a/jupyter_client/utils.py +++ b/jupyter_client/utils.py @@ -11,14 +11,14 @@ def run_sync(coro): def wrapped(*args, **kwargs): try: - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) import nest_asyncio # type: ignore nest_asyncio.apply(loop) - future = asyncio.ensure_future(coro(*args, **kwargs)) + future = asyncio.ensure_future(coro(*args, **kwargs), loop=loop) try: return loop.run_until_complete(future) except BaseException as e: diff --git a/pyproject.toml b/pyproject.toml index 9e6c94ebd..c50f85680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,3 +36,28 @@ testpaths = [ timeout = 300 # Restore this setting to debug failures # timeout_method = "thread" +asyncio_mode = "auto" +filterwarnings= [ + # Fail on warnings + "error", + + # Workarounds for https://github.com/pytest-dev/pytest-asyncio/issues/77 + "ignore:unclosed =5.5.6 +ipykernel>=6.5 ipython -jedi<0.18; python_version<="3.6" mypy pre-commit pytest -pytest-asyncio +pytest-asyncio>=0.18 pytest-cov pytest-timeout diff --git a/requirements.txt b/requirements.txt index 7c48249e9..880149ba6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ entrypoints jupyter_core>=4.9.2 -nest-asyncio>=1.5.1 -python-dateutil>=2.1 -pyzmq>=17 -tornado>=5.0 +nest-asyncio>=1.5.4 +python-dateutil>=2.8.2 +pyzmq>=22.3 +tornado>=6.0 traitlets