Dask_trial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c5d0b9e8-4cdd-4fcb-96fd-8b5a4359c1a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "OMP: Info #276: omp_set_nested routine deprecated, please use omp_set_max_active_levels instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initialization complete\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python3\n",
    "# inagler 12/09/23\n",
    "\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "import glob\n",
    "import pop_tools\n",
    "import dask.distributed\n",
    "\n",
    "path = '/Data/gfi/share/ModData/CESM2_LENS2/ocean/monthly/vvel/'\n",
    "files = glob.glob(path + '*.nc')\n",
    "\n",
    "grid_name = 'POP_gx1v7'\n",
    "\n",
    "region_defs = {\n",
    "    'NorthAtlantic':[{'match': {'REGION_MASK': [6]}, 'bounds': {'TLAT': [-20.0, 66.0]}}],\n",
    "    'LabradorSea': [{'match': {'REGION_MASK': [8]}, 'bounds': {'TLAT': [45.0, 66.0]}}],\n",
    "    'MediterraneanSea': [{'match': {'REGION_MASK': [7]}}]\n",
    "}\n",
    "mask3d = pop_tools.region_mask_3d(grid_name, region_defs=region_defs, mask_name='North Atlantic')\n",
    "mask3d = mask3d.sum('region')\n",
    "\n",
    "len_time = 3012\n",
    "dept_time_series_maxi = np.zeros((len_time, len(files)))\n",
    "dept_time_series_rapi = np.zeros((len_time, len(files)))\n",
    "dept_time_series_spgy = np.zeros((len_time, len(files)))\n",
    "\n",
    "print('initialization complete')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3c51c297-d6c9-49cd-964f-40353c74d27a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def depth_MOC(ds):\n",
    "    overturning_depth = (ds.VVEL * ds.dz * ds.DXU).sum(dim='nlon').cumsum(dim='z_t') * 1e-6\n",
    "    maxi = overturning_depth.isel(z_t=slice(27, 51)).max(dim=['nlat','z_t']).values\n",
    "    rapi = overturning_depth.isel(nlat=274, z_t=slice(27, 51)).max(dim='z_t').values\n",
    "    spgy = overturning_depth.isel(nlat=345, z_t=slice(27, 51)).max(dim='z_t').values\n",
    "    return maxi, rapi, spgy\n",
    "\n",
    "@dask.delayed\n",
    "def process_file(file):\n",
    "    ds = xr.open_dataset(file).where(mask3d == 1)\n",
    "    \n",
    "    # Update units to SI units\n",
    "    ds['VVEL'] *= 1e-2\n",
    "    ds['dz'] *= 1e-2\n",
    "    ds['z_t'] *= 1e-2\n",
    "    ds['z_w_top'] *= 1e-2\n",
    "    ds['z_w_bot'] *= 1e-2\n",
    "    ds['DXU'] *= 1e-2\n",
    "    ds['VVEL'].attrs['units'] = 'm/s'\n",
    "    ds['dz'].attrs['units'] = 'm'\n",
    "    ds['z_t'].attrs['units'] = 'm'\n",
    "    ds['z_w_top'].attrs['units'] = 'm'\n",
    "    ds['z_w_bot'].attrs['units'] = 'm'\n",
    "    ds['DXU'].attrs['units'] = 'm'\n",
    "    \n",
    "    return depth_MOC(ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "002dee24-8633-45e1-af27-4149f450c15f",
   "metadata": {},
   "source": [
    "These lines below are responsible for parallelizing the processing of multiple files using Dask. Let's break down what each line does:\n",
    "\n",
    "with `dask.distributed.Client()` as `client`:: This line creates a Dask client, which is used to manage distributed computing. When you create a Dask client, it connects to a Dask cluster (which can be local or distributed) and provides an interface for parallel and distributed computing. The with statement ensures that the client is properly closed when the block is exited.\n",
    "\n",
    "`results = dask.compute(*[process_file(file) for filein files], scheduler='threads'`: In this line, a list comprehension is used to create a list of delayed objects by calling the process_file(file) function for each file in the files list. Each `process_file(file) call is wrapped in a Dask delayed object, which represents a computation that can be executed lazily and in parallel.\n",
    "- `dask.compute` is then used to compute the results of all the delayed objects in parallel. The scheduler='threads' argument specifies that Dask should use a multi-threaded scheduler for parallel execution. This means that Dask will use multiple CPU threads to execute the delayed computations concurrently, which can speed up the processing of multiple \n",
    "- `he `*` operator is used to unpack the list of delayed objects and pass them as separate arguments to dask.compute.\n",
    "The results of the computations are stored in the results variable, which will be  tuple containing the results f - each delayed computation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9454e01c-097a-438a-b2fd-ff558accfd13",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/node.py:183: UserWarning: Port 8787 is already in use.\n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the HTTP server on port 35807 instead\n",
      "  warnings.warn(\n",
      "/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/dask/base.py:1367: UserWarning: Running on a single-machine scheduler when a distributed client is active might lead to unexpected results.\n",
      "  warnings.warn(\n",
      "2023-09-15 14:57:27,190 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,188 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,215 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,247 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,246 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,254 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,259 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,263 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:27,294 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,186 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,189 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,215 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,238 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,239 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,254 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,260 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,259 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,220 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,256 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,238 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,186 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,262 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:57:58,291 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,190 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 14:58:29,238 - distributed.worker - ERROR - Failed to communicate with scheduler during heartbeat.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/tcp.py\", line 225, in read\n",
      "    frames_nbytes = await stream.read_bytes(fmt_size)\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 456, in wait_for\n",
      "    return fut.result()\n",
      "asyncio.exceptions.CancelledError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 328, in connect\n",
      "    handshake = await asyncio.wait_for(comm.read(), time_left())\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/tasks.py\", line 458, in wait_for\n",
      "    raise exceptions.TimeoutError() from exc\n",
      "asyncio.exceptions.TimeoutError\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1215, in heartbeat\n",
      "    response = await retry_operation(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 386, in retry_operation\n",
      "    return await retry(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils_comm.py\", line 371, in retry\n",
      "    return await coro()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1160, in send_recv_from_rpc\n",
      "    comm = await self.pool.connect(self.addr)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1381, in connect\n",
      "    return await connect_attempt\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 1317, in _connect\n",
      "    comm = await connect(\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/comm/core.py\", line 333, in connect\n",
      "    raise OSError(\n",
      "OSError: Timed out during handshake while connecting to tcp://127.0.0.1:33888 after 30 s\n",
      "2023-09-15 15:12:42,993 - distributed.scheduler - WARNING - Worker failed to heartbeat within 300 seconds. Closing: <WorkerState 'tcp://127.0.0.1:37289', name: 4, status: running, memory: 0, processing: 0>\n",
      "2023-09-15 15:12:43,001 - distributed.scheduler - WARNING - Worker failed to heartbeat within 300 seconds. Closing: <WorkerState 'tcp://127.0.0.1:39699', name: 8, status: running, memory: 0, processing: 0>\n",
      "2023-09-15 15:12:43,002 - distributed.scheduler - WARNING - Worker failed to heartbeat within 300 seconds. Closing: <WorkerState 'tcp://127.0.0.1:42289', name: 1, status: running, memory: 0, processing: 0>\n",
      "2023-09-15 15:12:43,003 - distributed.scheduler - WARNING - Worker failed to heartbeat within 300 seconds. Closing: <WorkerState 'tcp://127.0.0.1:43562', name: 7, status: running, memory: 0, processing: 0>\n",
      "2023-09-15 15:12:43,004 - distributed.scheduler - WARNING - Worker failed to heartbeat within 300 seconds. Closing: <WorkerState 'tcp://127.0.0.1:45348', name: 2, status: running, memory: 0, processing: 0>\n",
      "2023-09-15 15:12:43,010 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'tcp://127.0.0.1:42289'.\n",
      "2023-09-15 15:12:43,012 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'tcp://127.0.0.1:45348'.\n",
      "2023-09-15 15:12:43,012 - distributed.worker - ERROR - Scheduler was unaware of this worker 'tcp://127.0.0.1:42289'. Shutting down.\n",
      "2023-09-15 15:12:43,014 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'tcp://127.0.0.1:39699'.\n",
      "2023-09-15 15:12:43,014 - distributed.worker - ERROR - Scheduler was unaware of this worker 'tcp://127.0.0.1:45348'. Shutting down.\n",
      "2023-09-15 15:12:43,015 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'tcp://127.0.0.1:37289'.\n",
      "2023-09-15 15:12:43,017 - distributed.scheduler - WARNING - Received heartbeat from unregistered worker 'tcp://127.0.0.1:43562'.\n",
      "2023-09-15 15:12:43,017 - distributed.worker - ERROR - Scheduler was unaware of this worker 'tcp://127.0.0.1:37289'. Shutting down.\n",
      "2023-09-15 15:12:43,017 - distributed.worker - ERROR - Scheduler was unaware of this worker 'tcp://127.0.0.1:39699'. Shutting down.\n",
      "2023-09-15 15:12:43,023 - distributed.worker - ERROR - Scheduler was unaware of this worker 'tcp://127.0.0.1:43562'. Shutting down.\n",
      "ERROR:asyncio.events:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils.py\", line 742, in wrapper\n",
      "    return await func(*args, **kwargs)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1497, in close\n",
      "    await self.finished()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 448, in finished\n",
      "    await self._event_finished.wait()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/locks.py\", line 214, in wait\n",
      "    await fut\n",
      "asyncio.exceptions.CancelledError\n",
      "ERROR:asyncio.events:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils.py\", line 742, in wrapper\n",
      "    return await func(*args, **kwargs)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1497, in close\n",
      "    await self.finished()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 448, in finished\n",
      "    await self._event_finished.wait()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/locks.py\", line 214, in wait\n",
      "    await fut\n",
      "asyncio.exceptions.CancelledError\n",
      "ERROR:asyncio.events:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/utils.py\", line 742, in wrapper\n",
      "    return await func(*args, **kwargs)\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/worker.py\", line 1497, in close\n",
      "    await self.finished()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/site-packages/distributed/core.py\", line 448, in finished\n",
      "    await self._event_finished.wait()\n",
      "  File \"/home/innag3580/.conda/envs/cartopy/lib/python3.10/asyncio/locks.py\", line 214, in wait\n",
      "    await fut\n",
      "asyncio.exceptions.CancelledError\n",
      "2023-09-15 15:12:45,293 - distributed.nanny - WARNING - Restarting worker\n",
      "2023-09-15 15:12:45,322 - distributed.nanny - ERROR - Worker process died unexpectedly\n",
      "2023-09-15 15:12:45,322 - distributed.nanny - ERROR - Worker process died unexpectedly\n",
      "2023-09-15 15:12:45,358 - distributed.nanny - WARNING - Restarting worker\n",
      "2023-09-15 15:12:45,639 - distributed.nanny - WARNING - Restarting worker\n",
      "2023-09-15 15:12:45,645 - distributed.nanny - WARNING - Restarting worker\n",
      "2023-09-15 15:12:45,701 - distributed.nanny - WARNING - Restarting worker\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Values of an IndexVariable are immutable and can not be modified inplace",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [4]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Use concurrent.futures to parallelize file processing\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m dask\u001b[38;5;241m.\u001b[39mdistributed\u001b[38;5;241m.\u001b[39mClient() \u001b[38;5;28;01mas\u001b[39;00m client:\n\u001b[0;32m----> 3\u001b[0m     results \u001b[38;5;241m=\u001b[39m \u001b[43mdask\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mprocess_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfile\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfiles\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscheduler\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mthreads\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/base.py:600\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(traverse, optimize_graph, scheduler, get, *args, **kwargs)\u001b[0m\n\u001b[1;32m    597\u001b[0m     keys\u001b[38;5;241m.\u001b[39mappend(x\u001b[38;5;241m.\u001b[39m__dask_keys__())\n\u001b[1;32m    598\u001b[0m     postcomputes\u001b[38;5;241m.\u001b[39mappend(x\u001b[38;5;241m.\u001b[39m__dask_postcompute__())\n\u001b[0;32m--> 600\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mschedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdsk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    601\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m repack([f(r, \u001b[38;5;241m*\u001b[39ma) \u001b[38;5;28;01mfor\u001b[39;00m r, (f, a) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(results, postcomputes)])\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/threaded.py:89\u001b[0m, in \u001b[0;36mget\u001b[0;34m(dsk, keys, cache, num_workers, pool, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(pool, multiprocessing\u001b[38;5;241m.\u001b[39mpool\u001b[38;5;241m.\u001b[39mPool):\n\u001b[1;32m     87\u001b[0m         pool \u001b[38;5;241m=\u001b[39m MultiprocessingPoolExecutor(pool)\n\u001b[0;32m---> 89\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mget_async\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     90\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpool\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msubmit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     91\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpool\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_max_workers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     92\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdsk\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     93\u001b[0m \u001b[43m    \u001b[49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     94\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     95\u001b[0m \u001b[43m    \u001b[49m\u001b[43mget_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_thread_get_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     96\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpack_exception\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpack_exception\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     97\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     98\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    100\u001b[0m \u001b[38;5;66;03m# Cleanup pools associated to dead threads\u001b[39;00m\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m pools_lock:\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/local.py:511\u001b[0m, in \u001b[0;36mget_async\u001b[0;34m(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)\u001b[0m\n\u001b[1;32m    509\u001b[0m         _execute_task(task, data)  \u001b[38;5;66;03m# Re-execute locally\u001b[39;00m\n\u001b[1;32m    510\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 511\u001b[0m         \u001b[43mraise_exception\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    512\u001b[0m res, worker_id \u001b[38;5;241m=\u001b[39m loads(res_info)\n\u001b[1;32m    513\u001b[0m state[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcache\u001b[39m\u001b[38;5;124m\"\u001b[39m][key] \u001b[38;5;241m=\u001b[39m res\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/local.py:319\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(exc, tb)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m exc\u001b[38;5;241m.\u001b[39m__traceback__ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m tb:\n\u001b[1;32m    318\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exc\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[0;32m--> 319\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/local.py:224\u001b[0m, in \u001b[0;36mexecute_task\u001b[0;34m(key, task_info, dumps, loads, get_id, pack_exception)\u001b[0m\n\u001b[1;32m    222\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    223\u001b[0m     task, data \u001b[38;5;241m=\u001b[39m loads(task_info)\n\u001b[0;32m--> 224\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43m_execute_task\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtask\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    225\u001b[0m     \u001b[38;5;28mid\u001b[39m \u001b[38;5;241m=\u001b[39m get_id()\n\u001b[1;32m    226\u001b[0m     result \u001b[38;5;241m=\u001b[39m dumps((result, \u001b[38;5;28mid\u001b[39m))\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/dask/core.py:119\u001b[0m, in \u001b[0;36m_execute_task\u001b[0;34m(arg, cache, dsk)\u001b[0m\n\u001b[1;32m    115\u001b[0m     func, args \u001b[38;5;241m=\u001b[39m arg[\u001b[38;5;241m0\u001b[39m], arg[\u001b[38;5;241m1\u001b[39m:]\n\u001b[1;32m    116\u001b[0m     \u001b[38;5;66;03m# Note: Don't assign the subtask results to a variable. numpy detects\u001b[39;00m\n\u001b[1;32m    117\u001b[0m     \u001b[38;5;66;03m# temporaries by their reference count and can execute certain\u001b[39;00m\n\u001b[1;32m    118\u001b[0m     \u001b[38;5;66;03m# operations in-place.\u001b[39;00m\n\u001b[0;32m--> 119\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m_execute_task\u001b[49m\u001b[43m(\u001b[49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcache\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43ma\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    120\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m ishashable(arg):\n\u001b[1;32m    121\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m arg\n",
      "Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36mprocess_file\u001b[0;34m(file)\u001b[0m\n\u001b[1;32m     13\u001b[0m ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mVVEL\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1e-2\u001b[39m\n\u001b[1;32m     14\u001b[0m ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdz\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1e-2\u001b[39m\n\u001b[0;32m---> 15\u001b[0m ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mz_t\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1e-2\u001b[39m\n\u001b[1;32m     16\u001b[0m ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mz_w_top\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1e-2\u001b[39m\n\u001b[1;32m     17\u001b[0m ds[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mz_w_bot\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1e-2\u001b[39m\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/xarray/core/_typed_ops.py:293\u001b[0m, in \u001b[0;36mDataArrayOpsMixin.__imul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    292\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__imul__\u001b[39m(\u001b[38;5;28mself\u001b[39m, other):\n\u001b[0;32m--> 293\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inplace_binary_op\u001b[49m\u001b[43m(\u001b[49m\u001b[43mother\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moperator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimul\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/xarray/core/dataarray.py:4382\u001b[0m, in \u001b[0;36mDataArray._inplace_binary_op\u001b[0;34m(self, other, f)\u001b[0m\n\u001b[1;32m   4380\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   4381\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcoords\u001b[38;5;241m.\u001b[39m_merge_inplace(other_coords):\n\u001b[0;32m-> 4382\u001b[0m         \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvariable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mother_variable\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   4383\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m MergeError \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m   4384\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m MergeError(\n\u001b[1;32m   4385\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAutomatic alignment is not supported for in-place operations.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   4386\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConsider aligning the indices manually or using a not-in-place operation.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   4387\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSee https://github.com/pydata/xarray/issues/3910 for more explanations.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   4388\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/xarray/core/_typed_ops.py:483\u001b[0m, in \u001b[0;36mVariableOpsMixin.__imul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    482\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__imul__\u001b[39m(\u001b[38;5;28mself\u001b[39m, other):\n\u001b[0;32m--> 483\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inplace_binary_op\u001b[49m\u001b[43m(\u001b[49m\u001b[43mother\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moperator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimul\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.conda/envs/cartopy/lib/python3.10/site-packages/xarray/core/variable.py:3062\u001b[0m, in \u001b[0;36mIndexVariable._inplace_binary_op\u001b[0;34m(self, other, f)\u001b[0m\n\u001b[1;32m   3061\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_inplace_binary_op\u001b[39m(\u001b[38;5;28mself\u001b[39m, other, f):\n\u001b[0;32m-> 3062\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\n\u001b[1;32m   3063\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mValues of an IndexVariable are immutable and can not be modified inplace\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3064\u001b[0m     )\n",
      "\u001b[0;31mTypeError\u001b[0m: Values of an IndexVariable are immutable and can not be modified inplace"
     ]
    }
   ],
   "source": [
    "# Use concurrent.futures to parallelize file processing\n",
    "with dask.distributed.Client() as client:\n",
    "    results = dask.compute(*[process_file(file) for file in files], scheduler='threads')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78db9743-dbc9-4a01-83b0-0c26f9efa490",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unpack results\n",
    "for i, result in enumerate(results):\n",
    "    dept_time_series_maxi[:, i], dept_time_series_rapi[:, i], dept_time_series_spgy[:, i] = result\n",
    "    print('file', i+1, '/', len(files), 'executed')\n",
    "\n",
    "print('computation finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e574a4b-3909-4097-b377-b4684a7ea4d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save time series to a single file\n",
    "np.save(\"timeseries/maxi_dens_time_series.npy\", dept_time_series_maxi)\n",
    "np.save(\"timeseries/rapi_dens_time_series.npy\", dept_time_series_rapi)\n",
    "np.save(\"timeseries/spgy_dens_time_series.npy\", dept_time_series_spgy)\n",
    "\n",
    "print('saving successful')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}