Merge pull request #29 from Ouranosinc/add-pavics-homepage-notebooks-…

…to-jenkins-testsuite

Add PAVICS homepage notebooks to Jenkins testsuite

Various changes so the notebooks pass Jenkins:
* Stable interface to setup the layout (unzip output.zip, .geojson files path are hardcoded) required by the notebooks.  With this stable interface, future layout change will be transparent to Jenkins
* Make all prod Thredds access go through httpS so Jenkins test server outside of Ouranos can still run the notebooks (CRIM)
* Add special markup for all Thredds url to get data directly from prod Pavics since that's too much `.ncml` (and `.nc`) files to replicate to all the test servers
* Disable output checking for failing output that is not critical to the notebooks so we do not need to refresh too often the notebook output

See each commit description for more detailed info.

Not regenerating all the .html files since the code change is really minimal (adding comments and one line switch the Thredds url from direct url to behind twitcher).

Matching PR that actually add the homepage notebooks to Jenkins: Ouranosinc/PAVICS-e2e-workflow-tests#79

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb

@@ -43,7 +43,7 @@
    ],
    "source": [
     "from siphon.catalog import TDSCatalog\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -777,6 +777,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "import xarray as xr\n",
     "\n",
     "# This does not download the entire dataset, just the metadata and attributes describing the content. \n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb

@@ -922,7 +922,7 @@
     "from IPython.display import display  # Fancy representation of xarray objects\n",
     "import warnings\n",
     "warnings.simplefilter('ignore')\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -3240,7 +3240,7 @@
     }
    ],
    "source": [
-    "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc'\n",
+    "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc'  # TEST_USE_PROD_DATA\n",
     "ds = xr.open_dataset(testurl, chunks=dict(time=1))\n",
     "fig = plt.figure(figsize=(20,6))\n",
     "ax1 = plt.subplot(1,2,1)\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb

@@ -777,7 +777,7 @@
     "from dask.diagnostics import ProgressBar\n",
     "warnings.simplefilter('ignore')\n",
     "# TODO change address\n",
-    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# Create Catalog\n",
     "cat = TDSCatalog(url)\n",
@@ -834,6 +834,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "freq = 'YS'\n",
     "print(f'calculating yearly output using freq=\"{freq}\"')\n",
     "with ProgressBar():\n",
@@ -1583,6 +1585,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "with xclim.set_options(metadata_locales=['fr']):\n",
     "    out_fr = atmos.tx_days_above(tasmax=ds.tasmax,\n",
     "                                thresh = '27 degC', \n",
@@ -1634,7 +1638,7 @@
     "from IPython.display import clear_output, display\n",
     "\n",
     "# cb-oura-1.0 thredds url\n",
-    "url = \"http://pavics.ouranos.ca:8083/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n",
+    "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"  # TEST_USE_PROD_DATA\n",
     "\n",
     "# specify output folder for netcdf files\n",
     "output = '/notebook_dir/writable-workspace/tmp/tutorial3/output'\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb

@@ -570,6 +570,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "from xclim import ensembles as xens\n",
     "from clisops.core import subset\n",
     "from pathlib import Path\n",
@@ -1134,6 +1136,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "# Calculate percentiles\n",
     "ds_perc = xens.ensemble_percentiles(ds_ens, values=[10, 50, 90], split=False)\n",
     "display(ds_perc)\n",

content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb

@@ -16500,6 +16500,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "from xclim import ensembles as xens\n",
     "import pandas as pd\n",
     "from clisops.core import subset\n",
@@ -17206,6 +17208,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "# 30 year means and delta calculations\n",
     "window = 30\n",
     "d30yAvg = ds_ens.rolling(year=window).mean()\n",
@@ -17663,6 +17667,8 @@
     }
    ],
    "source": [
+    "# NBVAL_IGNORE_OUTPUT\n",
+    "\n",
     "rolling = pn.widgets.FloatInput(value=30, step=10, start=10, end=30, width=50)\n",
     "rolling1 = pn.Column(pn.pane.Markdown('Smoothing'),rolling)\n",
     "\n",

content/notebooks/climate_indicators/setup_dirlayout.sh

@@ -0,0 +1,27 @@
+#!/bin/sh -x
+
+# Stable interface for the Jenkins testsuite to create any folders/files layout
+# for the notebooks to be tested under Jenkins.  With this stable interface,
+# Jenkins will not need to be modified for any future layout changes.
+#
+# Assume to be running inside the Jupyter env.
+
+THIS_FILE="`realpath "$0"`"
+THIS_DIR="`dirname "$THIS_FILE"`"
+
+# Unzip notebook 3 output to avoid having to generate it during automated
+# testing since it takes a long time.  Notebook 4 and 5 depend on this output.
+NOTEBOOK_3_OUTDIR_BASE="/notebook_dir/writable-workspace/tmp/tutorial3"
+mkdir -p $NOTEBOOK_3_OUTDIR_BASE
+unzip $THIS_DIR/output.zip -d $NOTEBOOK_3_OUTDIR_BASE/
+
+# Re-create /notebook_dir/pavics-homepage/tutorial_data layout for:
+# DriverError: /notebook_dir/pavics-homepage/tutorial_data/test_regions.geojson: No such file or directory
+#
+# Path to those .geojson files are hardcoded so users can copy the nb to
+# writable-workspace/ dir and still be able to run them seemlessly from
+# the Jupyter env (without having to also copy those *.geojson files with
+# the notebooks).
+HOMEPAGE_ROOT_DIR="/notebook_dir/pavics-homepage"
+mkdir -p $HOMEPAGE_ROOT_DIR
+ln -sv $THIS_DIR/tutorial_data $HOMEPAGE_ROOT_DIR/tutorial_data