GFTS_kerchunk-NORTHWESTSHELF_ANALYSIS_FORECAST_PHY.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5839b3de-d6b4-4fd5-a771-a25485b7be64",
   "metadata": {},
   "source": [
    "# List dataset we need for GFTS\n",
    "unfortunately, the dataset we use is not in zarr at copernicus marine.\n",
    "We can download (keeping the directory format as it is) and make kerchunk \n",
    "and create intake catalogue.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86e5a787-c78e-40d9-8588-8f88a610bd41",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install copernicusmarine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b665368e-34e4-42d8-851d-ea8da549bce9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1.0\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "import xarray as xr\n",
    "import copernicusmarine as copernicusmarine\n",
    "import pandas as pd\n",
    "print(copernicusmarine.__version__)\n",
    "#only for the first time: \n",
    "#copernicusmarine.login() \n",
    "pd.set_option('display.max_colwidth', None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52993e83-5349-45fd-97f0-305eb387078a",
   "metadata": {},
   "source": [
    "## Download the copernicus catalogue containing ['cmems_mod_nws_phy_anfc_0.027deg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7320f930-e473-43d9-acbd-23a1d21eaf8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Fetching catalog: 100%|██████████| 3/3 [00:09<00:00,  3.27s/it]\n"
     ]
    }
   ],
   "source": [
    "catalogue = copernicusmarine.describe(\n",
    "    include_datasets=True,\n",
    "#    contains = ['European North West Shelf - Ocean Physics '],\n",
    "#     contains = ['Atlantic-Iberian Biscay Irish- Ocean Physics']   \n",
    "    contains = ['cmems_mod_nws_phy_anfc_0.027deg']\n",
    "#contains=['cmems_mod_ibi_phy_anfc_0.027deg','cmems_mod_nws_phy_anfc_0.027deg']\n",
    "#\n",
    "#    contains=['Ocean Physics']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e7c0257-5f13-40e4-8c6e-77067eb36fad",
   "metadata": {},
   "source": [
    "## find the dataset_id and path for ['cmems_mod_nws_phy_anfc_0.027deg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a7edea6a-5be9-420a-9cfb-fefae0fb2321",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Atlantic - European North West Shelf - Ocean Physics Analysis and Forecast NWSHELF_ANALYSISFORECAST_PHY_004_013\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-2D_PT15M-i {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] default None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT15M-i_202309\n",
      "      1 arco-geo-series https://s3.waw3-1.cloudferro.com/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT15M-i_202309/timeChunked.zarr\n",
      "      2 arco-time-series https://s3.waw3-1.cloudferro.com/mdl-arco-geo-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT15M-i_202309/geoChunked.zarr\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-2D_PT1H-m {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] default None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT1H-m_202309\n",
      "      1 arco-geo-series https://s3.waw3-1.cloudferro.com/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT1H-m_202309/timeChunked.zarr\n",
      "      2 arco-time-series https://s3.waw3-1.cloudferro.com/mdl-arco-geo-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-2D_PT1H-m_202309/geoChunked.zarr\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-3D_P1D-m {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] default None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1D-m_202309\n",
      "      1 arco-geo-series https://s3.waw3-1.cloudferro.com/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1D-m_202309/timeChunked.zarr\n",
      "      2 arco-time-series https://s3.waw3-1.cloudferro.com/mdl-arco-geo-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1D-m_202309/geoChunked.zarr\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-3D_P1M-m {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] default None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1M-m_202309\n",
      "      1 arco-geo-series https://s3.waw3-1.cloudferro.com/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1M-m_202309/timeChunked.zarr\n",
      "      2 arco-time-series https://s3.waw3-1.cloudferro.com/mdl-arco-geo-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_P1M-m_202309/geoChunked.zarr\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-3D_PT1H-m {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] default None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_PT1H-m_202309\n",
      "      1 arco-geo-series https://s3.waw3-1.cloudferro.com/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_PT1H-m_202309/timeChunked.zarr\n",
      "      2 arco-time-series https://s3.waw3-1.cloudferro.com/mdl-arco-geo-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_PT1H-m_202309/geoChunked.zarr\n",
      "dataset_id cmems_mod_nws_phy_anfc_0.027deg-3D_static {}\n",
      "versions['label'] 202309\n",
      "parts['name','retired-date'] bathy None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309/NWS-MFC_004_013_mask_bathy.nc\n",
      "      1 static-arco https://s3.waw3-1.cloudferro.com:443/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309--ext--bathy/static.zarr\n",
      "parts['name','retired-date'] coords None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309/NWS-MFC_004_013_coordinates.nc\n",
      "      1 static-arco https://s3.waw3-1.cloudferro.com:443/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309--ext--coords/static.zarr\n",
      "parts['name','retired-date'] mdt None\n",
      "      0 original-files https://s3.waw3-1.cloudferro.com/mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309/NWS-MFC_004_013_mdt.nc\n",
      "      1 static-arco https://s3.waw3-1.cloudferro.com:443/mdl-arco-time-041/arco/NWSHELF_ANALYSISFORECAST_PHY_004_013/cmems_mod_nws_phy_anfc_0.027deg-3D_static_202309--ext--mdt/static.zarr\n"
     ]
    }
   ],
   "source": [
    "uris={}\n",
    "for index, product in enumerate(catalogue['products']):\n",
    "    print(index, product['title'], product['product_id'])    \n",
    "    for value in catalogue['products'][index]['datasets']:\n",
    "        dataset_id = value['dataset_id']\n",
    "        uris[dataset_id]={}\n",
    "        print('dataset_id',dataset_id,uris[dataset_id])\n",
    "        for versions_id,versions in  enumerate(value['versions']):\n",
    "            print(\"versions['label']\",versions['label'],)\n",
    "            for parts_id, parts in  enumerate( versions['parts']):\n",
    "                print(\"parts['name','retired-date']\",parts['name'],parts['retired_date'],)\n",
    "                for service_id,service in  enumerate( parts['services']):\n",
    "                    service_name = service['service_type']['service_name']\n",
    "                    uris[dataset_id][service_name]=service['uri']\n",
    "                    print('     ',service_id,service_name,service['uri'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8a652c2c-5595-4004-a445-8bdf8230f760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_WAV_004_014',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_OMI_SL_area_averaged_anomalies',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_OMI_TEMPSAL_extreme_var_temp_mean_and_anomaly',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_OMI_TEMPSAL_sst_area_averaged_anomalies',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_OMI_TEMPSAL_sst_trend',\n",
       " 'mdl-native-13/native/NWSHELF_ANALYSISFORECAST_BGC_004_002',\n",
       " 'mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_004_013',\n",
       " 'mdl-native-13/native/NWSHELF_ANALYSISFORECAST_PHY_LR_004_001',\n",
       " 'mdl-native-13/native/NWSHELF_ANALYSISFORECAST_WAV_004_014',\n",
       " 'mdl-native-13/native/NWSHELF_MULTIYEAR_BGC_004_011',\n",
       " 'mdl-native-13/native/NWSHELF_MULTIYEAR_PHY_004_009',\n",
       " 'mdl-native-13/native/NWSHELF_REANALYSIS_PHY_ENS_004_016',\n",
       " 'mdl-native-13/native/NWSHELF_REANALYSIS_WAV_004_015']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xarray as xr\n",
    "import s3fs\n",
    "\n",
    "url='/mdl-native-13/native/'\n",
    "\n",
    "#url='s3://mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2019/01/metoffice_foam1_amm15_NWS_SSH_b20190103_hi20190101.nc'\n",
    "fs=s3fs.S3FileSystem(anon=True, \n",
    "                  endpoint_url=\"https://s3.waw3-1.cloudferro.com\",\n",
    "                     )  \n",
    "fs.ls(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef39b3dd-e473-4dc3-805d-b1135210c4ed",
   "metadata": {},
   "source": [
    "## The onoe we are using is **NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ed0abd6f-3029-4191-8af6-303400ff8232",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-001-013-STATIC_202112',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-dm-BED_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-dm-CUR_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-dm-MLD_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-dm-SAL_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-dm-TEM_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-BED_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-CUR_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-MLD_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SAL_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSC_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSS_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SST_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-qh-SSC_202012',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-qh-SSH_202012']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fs.ls(url+'NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5049a8af-28b7-4b3d-9b09-d0de8adb0234",
   "metadata": {},
   "source": [
    "## we use \n",
    "- hi-TEM_202012\n",
    "- hi-SSH_202012\n",
    "- 001-013-STATIC_202112"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e46c8686-bc4a-47b9-8f0b-171cd253f5f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2019',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2020',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2021',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2022',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2023',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM_202012/2024']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_url_TEM=url+'NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-'+'TEM'+'_202012'\n",
    "fs.ls(full_url_TEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "609acbed-0613-4243-8323-732f1e6301ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2019',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2020',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2021',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2022',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2023',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-SSH_202012/2024']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_url_SSH=url+'NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-'+'SSH'+'_202012'\n",
    "fs.ls(full_url_SSH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d4669d1d-6fd3-4975-a06e-24620c6861c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-001-013-STATIC_202112/NWS-MFC_004_013_mask_bathy.nc',\n",
       " 'mdl-native-13/native/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-001-013-STATIC_202112/NWS-MFC_004_013_mdt.nc']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_url_static=url+'NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-001-013-STATIC_202112'\n",
    "fs.ls(full_url_static)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "112b8f5c-0d1d-4595-8bca-603dc714d5ec",
   "metadata": {},
   "source": [
    "#  Below is old workflow, for s3, we would only need to make one catalogue which is based on gfts-dataref space"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db20d7ff-e10a-4e98-b6d7-fdb2eed546c1",
   "metadata": {},
   "source": [
    "## Starting Dask cluster on HPC\n",
    "\n",
    "Please refer to the [Dask-hpcconfig datarmor example Jupyter notebook](https://github.com/umr-lops/dask-hpcconfig/tree/main/docs/tutorials) to understand what the next three cells mean.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c74bc43f-975e-4600-a41c-5402ad490fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask_hpcconfig\n",
    "from distributed import Client"
   ]
  },
  {
   "cell_type": "raw",
   "id": "1b842f77-ded8-403d-857e-f529a9fad86e",
   "metadata": {
    "tags": []
   },
   "source": [
    "overrides = {}  # ,\"cluster.c\": n_worker_per_node }\n",
    "\n",
    "cluster = dask_hpcconfig.cluster(\"datarmor-local\", **overrides)\n",
    "client = Client(cluster)\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "963bb236-b765-4a7d-a8ed-b7ade53696ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "overrides = {}\n",
    "# overrides = { \"cluster.cores\": 28 , \"cluster.processes\": 6 }\n",
    "\n",
    "cluster = dask_hpcconfig.cluster(\"datarmor\", **overrides)\n",
    "client = Client(cluster)\n",
    "cluster.scale(20)\n",
    "client\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0447755-3dc0-46fb-af15-7b4ca045102a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!qstat -u todaka"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01627faf-6617-4b2a-acb9-2554564a65d4",
   "metadata": {},
   "source": [
    "## Converting Multiple NetCDF Files to a Kerchunk Catalogue\n",
    "\n",
    "We use `kerchunk.combine.auto_dask`  with `kerchunk.hdf.SingleHdf5ToZarr`  \n",
    "\n",
    "`kerchunk.combine.auto_dask`  convert each NetCDF files into kerchunk catalogue and concatenate them to one kerchunk catalogue all at once, in parallel using dask. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "520b5ba2-57dc-4ca0-954a-479149bbdef1",
   "metadata": {},
   "source": [
    "## We will use [Atlantic - European North West Shelf - Ocean Physics Analysis and Forecast from Copernicus Marine services](https://data.marine.copernicus.eu/product/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/services) \n",
    "\n",
    "This data has different filename for same date range. (below tagged as 'tag')\n",
    "We open each file and see if we can make concatenation easily.\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "4517f153-a248-4cfd-8d4f-3462e1a1d6a5",
   "metadata": {},
   "source": [
    "xr.open_mfdata-set(newpath[1],chunks={}).mdt.plot()#,engine='h5netcdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ed204ec-d534-4ea3-873d-d39bd861242a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Transformation of \n",
    "#/home/datawork-taos-s/public/data_tmp/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/NWS-MFC_004_013_mdt.nc\n",
    "\n",
    "import glob\n",
    "import xarray as xr\n",
    "\n",
    "\n",
    "base_url='/home/datawork-taos-s/public/data_tmp/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/'\n",
    "\n",
    "\n",
    "years=[str(i)  for i in range(2022,2023)]\n",
    "months=[\"0\"+str(i)  for i in range(6,9)]\n",
    "\n",
    "tags=[ ]\n",
    "\n",
    "\n",
    "file_paths=[]\n",
    "\n",
    "file_pattern = \"*.nc\"\n",
    "newpath=glob.glob(dir_url + file_pattern)\n",
    "print(newpath)\n",
    "display(xr.open_mfdataset(newpath,chunks={}))#,engine='h5netcdf'))\n",
    "file_paths.extend([newpath[0]])\n",
    "#print(dir_url + file_pattern,(glob.glob(dir_url + file_pattern)))\n",
    "#file_paths.extend(glob.glob(dir_url + file_pattern))\n",
    "            #print(dir_url + file_pattern)\n",
    "#print(file_paths)\n",
    "def translate_dask(file):\n",
    "    url = \"file://\" + file\n",
    "    #print(\"working on \", file)\n",
    "    with fsspec.open(url) as inf:\n",
    "        h5chunks = SingleHdf5ToZarr(inf, url, inline_threshold=100)\n",
    "        return h5chunks.translate()\n",
    "\n",
    "\n",
    "b = db.from_sequence(file_paths)\n",
    "result_indask = b.map(translate_dask)\n",
    "a = result_indask.compute()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54594bde-a80c-4d2d-9bc5-ec175887dafa",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "#transformation of 3D value, then 2D value\n",
    "\n",
    "import glob\n",
    "import xarray as xr\n",
    "\n",
    "\n",
    "base_url='/home/datawork-taos-s/public/data_tmp/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/'\n",
    "\n",
    "\n",
    "years=[str(i)  for i in range(2022,2023)]\n",
    "months=[\"0\"+str(i)  for i in range(6,9)]\n",
    "\n",
    "\n",
    "tags=[\"CUR\",  \"SAL\",  \"TEM\", ]\n",
    "tags=[\"SSS\", \"SST\", \"SSC\",\"BED\",\"MLD\",\"SSH\",]\n",
    "\n",
    "\n",
    "\n",
    "file_paths=[]\n",
    "for tag in tags:\n",
    "    for year in years:\n",
    "        newpath=[]\n",
    "        for month in months:\n",
    "            dir_url=base_url\n",
    "            file_pattern = year+\"/\" + month + \"/*\"+ tag +\"*.nc\"\n",
    "            newpath.extend(glob.glob(dir_url + file_pattern))\n",
    "    print([newpath[0]])\n",
    "    display(xr.open_mfdataset([newpath[0]],chunks={},engine='h5netcdf'))\n",
    "    file_paths.extend(newpath)\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb36b84a-1bc1-4680-82ff-4c5ab5174e4e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Method 1\n",
    "\n",
    "We use `kerchunk.hdf.SingleHdf5ToZarr` with `dask.bag` to convert each NetCDF file to kerchunk catalogs, and then concatenate them with `kerchunk.combine.MultiZarrToZarr` to create a single kerchunk catalog. This workflow is used in the first version of auto-kerchunk.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26021445-8f81-498a-91e5-92803ed27761",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "import dask.bag as db\n",
    "import fsspec\n",
    "from kerchunk.hdf import SingleHdf5ToZarr\n",
    "\n",
    "\n",
    "def translate_dask(file):\n",
    "    url = \"file://\" + file\n",
    "    #print(\"working on \", file)\n",
    "    with fsspec.open(url) as inf:\n",
    "        h5chunks = SingleHdf5ToZarr(inf, url, inline_threshold=100)\n",
    "        return h5chunks.translate()\n",
    "\n",
    "\n",
    "b = db.from_sequence(file_paths)\n",
    "result_indask = b.map(translate_dask)\n",
    "result = result_indask.compute()\n",
    "\n",
    "from kerchunk.combine import MultiZarrToZarr\n",
    "\n",
    "mzz = MultiZarrToZarr(\n",
    "    result,\n",
    "    concat_dims=[\"time\"],\n",
    ")\n",
    "a = mzz.translate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21952fa2-81da-4ec1-9ceb-2feccb62de38",
   "metadata": {},
   "source": [
    "### Method 2\n",
    "\n",
    "We use `kerchunk.combine.auto_dask`  instead of `kerchunk.combine.MultiZarrToZarr`  as described above.\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "2e59df76-888b-4e37-8879-a23c27bede78",
   "metadata": {
    "tags": []
   },
   "source": [
    "%%time\n",
    "import dask.bag as db\n",
    "import fsspec\n",
    "\n",
    "from kerchunk.combine import auto_dask\n",
    "from kerchunk.hdf import SingleHdf5ToZarr\n",
    "\n",
    "\n",
    "class PassThrough:\n",
    "    def __init__(self, refs):\n",
    "        self.refs = refs\n",
    "\n",
    "    def translate(self):\n",
    "        return self.refs\n",
    "\n",
    "\n",
    "def translate_dask(file):\n",
    "    url = \"file://\" + file\n",
    "    print(\"working on \", file)\n",
    "    with fsspec.open(url) as inf:\n",
    "        h5chunks = SingleHdf5ToZarr(inf, url, inline_threshold=100)\n",
    "        return h5chunks.translate()\n",
    "\n",
    "\n",
    "b = db.from_sequence(file_paths)\n",
    "result_indask = b.map(translate_dask)\n",
    "result = result_indask.compute()\n",
    "\n",
    "b = auto_dask(\n",
    "    result,\n",
    "    single_driver=PassThrough,\n",
    "    # single_driver=JustLoad,\n",
    "    single_kwargs={},  # \"storage_options\": {\"anon\": False}},\n",
    "    mzz_kwargs={\"concat_dims\": [\"time\"]},\n",
    "    n_batches=3,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c16c44ea-2e3f-419b-a6e6-9bf902244734",
   "metadata": {},
   "source": [
    "### Method 3. \n",
    "\n",
    "We use `kerchunk.combine.auto_dask`  with `kerchunk.hdf.SingleHdf5ToZarr`  \n",
    "\n",
    "`kerchunk.combine.auto_dask`  convert each NetCDF files into kerchunk catalogue and concatenate them to one kerchunk catalogue all at once, in parallel using dask.   "
   ]
  },
  {
   "cell_type": "raw",
   "id": "286c5fab-0a85-4c20-b485-508cfc66c2c1",
   "metadata": {
    "tags": []
   },
   "source": [
    "%%time\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from kerchunk.combine import auto_dask  # , SingleHdf5ToZarr\n",
    "from kerchunk.hdf import SingleHdf5ToZarr\n",
    "\n",
    "c = auto_dask(\n",
    "    file_paths,\n",
    "    single_driver=SingleHdf5ToZarr,\n",
    "    # single_driver=JustLoad,\n",
    "    single_kwargs={},  # \"storage_options\": {\"anon\": False}},\n",
    "    mzz_kwargs={\"concat_dims\": [\"time\"]},\n",
    "    n_batches=3,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb69fbd-a906-4621-8bf1-13820b991a71",
   "metadata": {},
   "source": [
    "## Loading data to Xarray using kerchunk and verify."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a391fddf-89d6-4a61-b97c-46c9403202bb",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "import xarray as xr\n",
    "\n",
    "test = xr.open_dataset(\n",
    "    \"reference://\",\n",
    "    engine=\"zarr\",\n",
    "    backend_kwargs={\n",
    "        \"storage_options\": {\n",
    "            \"fo\": a,\n",
    "        },\n",
    "        \"consolidated\": False,\n",
    "    },\n",
    "    chunks={},\n",
    ")\n",
    "test"
   ]
  },
  {
   "cell_type": "raw",
   "id": "33bc445d-1b58-4eca-9ea6-90abb9fc0917",
   "metadata": {},
   "source": [
    "#3D plot better to use inspect from hvplot?\n",
    "\n",
    "import hvplot.xarray\n",
    "lat=48.43\n",
    "lon=-5.021\n",
    "test.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).uo.plot()\n",
    "test.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).vo.plot()\n",
    "test.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).so.plot()\n",
    "test.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).thetao.plot()\n",
    "\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "0257b987-fe04-469d-b24b-da3fa4e70a97",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "#2D plot better to use inspect from hvplot?\n",
    "import hvplot.xarray\n",
    "lat=48.43\n",
    "lon=-5.021\n",
    "test=test.sel().sel(lat=lat,lon=lon,method='nearest')\n",
    "test.zos.plot()\n",
    "test.thetao.plot()\n",
    "test.so.plot()\n",
    "test.bottomT.plot()\n",
    "test.mlotst.plot()\n",
    "test.uo.plot()\n",
    "test.vo.plot()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "d7205ba5-bca3-4beb-9cba-b1bfdd2732f0",
   "metadata": {},
   "source": [
    "#mdt plot better to use inspect from hvplot?\n",
    "import hvplot.xarray\n",
    "\n",
    "test.mask.isel(depth=20).plot()\n",
    "test.deptho_lev_interp.plot()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00c3e451-4553-40c5-b076-37616178e470",
   "metadata": {},
   "source": [
    "## Publishing the kerchunk for the usage from HPC \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b95caeb-4bd7-46c3-8663-81c3605bd59a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "#base_url='/home/ref-cersat-public/sea-surface-temperature/odyssea/'\n",
    "base_url='/home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/'\n",
    "#base_url='/home/datawork-taos-s/public/data/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM/'\n",
    "\n",
    "name='datatmp_mdt'\n",
    "name='datatmp_2022_3D'\n",
    "name='datatmp_2022_2D'\n",
    "\n",
    "\n",
    "name=base_url+name.replace('/', '_')\n",
    "os.makedirs(name, exist_ok=True)\n",
    "print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "195ed466-105d-48d0-b423-f0fb84c96144",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import json\n",
    "jsonfile=name+'.json.zstd'#+'IFR-L4_HRSST-SSTfnd-ODYSSEA-ATL_002-v02.1-fv01.0.json.zstd'\n",
    "storage_options_in= {\"compression\": \"zstd\"}\n",
    "with open(jsonfile, mode='w') as f :\n",
    "    json.dump(a, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2178f2a0-ecc9-480b-92dc-b8cc6ef933b7",
   "metadata": {},
   "source": [
    "# make sure to do above json creation for each group of kerchunk file (2D, 3D, mdt) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1d0bc67-53e0-46ad-b8ef-9342149c7c22",
   "metadata": {},
   "source": [
    "## Publishing the kerchunk for the usage from Cloud \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f12f7ed4-13e9-4837-87c7-65ba0817c85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "names=!find /home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/ |grep json.zst |grep datatmp\n",
    "#names=[names[0]]\n",
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cd7865a-174d-48b4-9dbf-2d8bb27bb36b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "paths= [ os.path.dirname(name)   for name in names]\n",
    "paths=list(set(paths))\n",
    "def createpath(path):\n",
    "    newpath=path.replace('intranet', 'public')\n",
    "    return os.makedirs(newpath, exist_ok=True)\n",
    "createpath= [ createpath(path)   for path in paths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f46b32ec-c57b-4e56-a4bf-99e0a391f6ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import xarray as xr\n",
    "test= xr.open_dataset(\n",
    "    \"reference://\", engine=\"zarr\",\n",
    "    backend_kwargs={\n",
    "        \"storage_options\": {\n",
    "            \"fo\":'file://'+names[0],\n",
    "        },\n",
    "        \"consolidated\": False\n",
    "    },chunks={}\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a3c91c3-1952-4c6c-8ea0-e9ef1c553bb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "lat=48.43\n",
    "lon=-5.021\n",
    "test.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).thetao.plot()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3614fa54-ac13-4af7-8100-7a347ee06ccc",
   "metadata": {},
   "source": [
    "## Translate kerchunk catalogue for intranet access to https access"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaef0c11-e816-4604-97e4-b2157ca2a44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import fsspec\n",
    "import ujson\n",
    "import dask\n",
    "\n",
    "\n",
    "\n",
    "def match_keys(mapping, value):\n",
    "    for k in mapping:\n",
    "        if k in value: \n",
    "            return k\n",
    "        \n",
    "    raise ValueError(f\"could not find {value} in mapping\") \n",
    "    \n",
    "def match_in_keys(mapping,value):\n",
    "    try:\n",
    "        match_keys(mapping,value)\n",
    "        return True\n",
    "    except ValueError:\n",
    "        return False    \n",
    "\n",
    "def rename_target(refs, renames):\n",
    "    #from kerchunk.utils import conslidate\n",
    "    print('in rename_target')\n",
    "    \"\"\"Utility to change URLs in a reference set in a predictable way\n",
    "\n",
    "    For reference sets including templates, this is more easily done by\n",
    "    using template overrides at access time; but rewriting the references\n",
    "    and saving a new file means not having to do that every time.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    refs: dict\n",
    "        Reference set\n",
    "    renames: dict[str, str]\n",
    "        Mapping from the old URL (including protocol, if this is how they appear\n",
    "        in the original) to new URL\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    dict: the altered reference set, which can be saved\n",
    "    \"\"\"\n",
    "    fs = fsspec.filesystem(\"reference\", fo=refs)  # to produce normalised refs\n",
    "    refs = fs.references\n",
    "    out = {}\n",
    "    for k, v in refs.items():\n",
    "        if isinstance(v, list) and v[0] in renames:\n",
    "            out[k] = [renames[v[0]]] + v[1:]\n",
    "        elif isinstance(v, list) and match_in_keys(renames, v[0]) :\n",
    "            url = v[0]\n",
    "            #print(url)\n",
    "            key = match_keys(renames, url)\n",
    "            new_url = url.replace(key, renames[key])\n",
    "            out[k] = [new_url] + v[1:]\n",
    "            #print(new_url)\n",
    "        else:\n",
    "            out[k] = v\n",
    "        #    print('boo')\n",
    "    return consolidate(out)\n",
    "\n",
    "def rename_target_files(\n",
    "    url_in, renames, url_out=None, storage_options_in=None, storage_options_out=None):\n",
    "    #print('in rename_target_files',url_in,renames,url_out,storage_options_in, storage_options_out)\n",
    "\n",
    "    \"\"\"Perform URL renames on a reference set - read and write from JSON\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    url_in: str\n",
    "        Original JSON reference set\n",
    "    renames: dict\n",
    "        URL renamings to perform (see ``renate_target``)\n",
    "    url_out: str | None\n",
    "        Where to write to. If None, overwrites original\n",
    "    storage_options_in: dict | None\n",
    "        passed to fsspec for opening url_in\n",
    "    storage_options_out: dict | None\n",
    "        passed to fsspec for opening url_out. If None, storage_options_in is used.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    None\n",
    "    \"\"\"\n",
    "    with fsspec.open(url_in, **(storage_options_in or {})) as f:\n",
    "        print(url_in,storage_options_in)\n",
    "        old = ujson.load(f)\n",
    "    new = rename_target(old, renames)\n",
    "    if url_out is None:\n",
    "        url_out = url_in\n",
    "    if storage_options_out is None:\n",
    "        storage_options_out = storage_options_in\n",
    "    with fsspec.open(url_out, mode=\"wt\", **(storage_options_out or {})) as f:\n",
    "        ujson.dump(new, f)\n",
    "\n",
    "def consolidate(refs):\n",
    "    \"\"\"Turn raw references into output\"\"\"\n",
    "    out = {}\n",
    "    for k, v in refs.items():\n",
    "        if isinstance(v, bytes):\n",
    "            try:\n",
    "                # easiest way to test if data is ascii\n",
    "                out[k] = v.decode(\"ascii\")\n",
    "            except UnicodeDecodeError:\n",
    "                out[k] = (b\"base64:\" + base64.b64encode(v)).decode()\n",
    "        else:\n",
    "            out[k] = v\n",
    "    return {\"version\": 1, \"refs\": out}\n",
    "\n",
    "#@dask.delayed\n",
    "def translate(name):\n",
    "    in_path='file:///home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM/'\n",
    "    in_path='file:///home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/'\n",
    "\n",
    "    out_path='file:///home/datawork-taos-s/public/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM/'\n",
    "    out_path='file:///home/datawork-taos-s/public/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/'\n",
    "    name=name.replace('/home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/MetO-NWS-PHY-hi-TEM/','')\n",
    "    name=name.replace('/home/datawork-taos-s/intranet/kerchunk/NORTHWESTSHELF_ANALYSIS_FORECAST_PHY_004_013/','')\n",
    "    url_in = in_path+name\n",
    "    url_out = out_path+name\n",
    "\n",
    "    renames={'file:///home/datawork-taos-s/public/':'https://data-taos.ifremer.fr/'} \n",
    "\n",
    "    storage_options_in= {}#{\"compression\": \"zstd\"}\n",
    "    storage_options_out= {\"compression\": \"zstd\"}\n",
    "    return rename_target_files(\n",
    "        url_in, renames, url_out, storage_options_in=storage_options_in\n",
    "        , storage_options_out=storage_options_out ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d34b3d14-b169-402c-aacf-d493b08a165b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "translated= [ translate(name)   for name in names]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46006ed0-0481-4764-a87f-8f3bfb64b9a2",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "Create intake catalogue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f7ba0e3-da06-4920-a157-bb9282e76db7",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cat /home/datawork-taos-s/public/kerchunk/ref-copernicus.yaml\n",
    "!cat /home/datawork-taos-s/intranet/kerchunk/ref-copernicus.yaml\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1c0ee24-2eb9-4a4a-9907-0262075f2e09",
   "metadata": {},
   "outputs": [],
   "source": [
    "import fsspec\n",
    "import intake\n",
    "import xarray as xr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "161dc775-bbaf-4ca1-bbbf-4cf99bfdd1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "catalogue = \"/home/datawork-taos-s/intranet/kerchunk/ref-copernicus.yaml\"\n",
    "#catalogue = \"https://data-taos.ifremer.fr/kerchunk/ref-copernicus.yaml\"\n",
    "#catalogue = \"https://data-taos.ifremer.fr/kerchunk/ref-cersat.yaml\"\n",
    "\n",
    "cat = intake.open_catalog(catalogue)\n",
    "cat.data_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "626c1934-be9e-4067-a47b-5030f7f57b16",
   "metadata": {},
   "outputs": [],
   "source": [
    "test=cat.data_tmp(type='mdt').to_dask()\n",
    "lat=48.43\n",
    "lon=-5.021\n",
    "test#.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).thetao.plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dd5a4d2-8eee-437b-8fab-58ff32331451",
   "metadata": {},
   "outputs": [],
   "source": [
    "test=cat.data_tmp(year='2022').to_dask()\n",
    "lat=48.43\n",
    "lon=-5.021\n",
    "test#.sel().sel(lat=lat,lon=lon,method='nearest',depth=0).thetao.plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf524495-a31d-40b1-af23-6ef4d6e1ce9c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}