From da44b397bcb8e98d7f890bf5062a6aca22aae1f4 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 12:52:59 -0400 Subject: [PATCH 1/9] jenkins: add stable setup interface for jenkins --- .../climate_indicators/setup_dirlayout.sh | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 content/notebooks/climate_indicators/setup_dirlayout.sh diff --git a/content/notebooks/climate_indicators/setup_dirlayout.sh b/content/notebooks/climate_indicators/setup_dirlayout.sh new file mode 100755 index 0000000..8e61fab --- /dev/null +++ b/content/notebooks/climate_indicators/setup_dirlayout.sh @@ -0,0 +1,27 @@ +#!/bin/sh -x + +# Stable interface for the Jenkins testsuite to create any folders/files layout +# for the notebooks to be tested under Jenkins. With this stable interface, +# Jenkins will not need to be modified for any future layout changes. +# +# Assume to be running inside the Jupyter env. + +THIS_FILE="`realpath "$0"`" +THIS_DIR="`dirname "$THIS_FILE"`" + +# Unzip notebook 3 output to avoid having to generate it during automated +# testing since it takes a long time. Notebook 4 and 5 depend on this output. +NOTEBOOK_3_OUTDIR_BASE="/notebook_dir/writable-workspace/tmp/tutorial3" +mkdir -p $NOTEBOOK_3_OUTDIR_BASE +unzip $THIS_DIR/output.zip -d $NOTEBOOK_3_OUTDIR_BASE/ + +# Re-create /notebook_dir/pavics-homepage/tutorial_data layout for: +# DriverError: /notebook_dir/pavics-homepage/tutorial_data/test_regions.geojson: No such file or directory +# +# Path to those .geojson files are hardcoded so users can copy the nb to +# writable-workspace/ dir and still be able to run them seemlessly from +# the Jupyter env (without having to also copy those *.geojson files with +# the notebooks). +HOMEPAGE_ROOT_DIR="/notebook_dir/pavics-homepage" +mkdir -p $HOMEPAGE_ROOT_DIR +ln -sv $THIS_DIR/tutorial_data $HOMEPAGE_ROOT_DIR/tutorial_data From 8fe155d267f20a8854e83eb216ec4d425fd1995f Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 13:14:04 -0400 Subject: [PATCH 2/9] jenkins: ignore execution duration output in notebook 3 To fix this error in Jenkins: ``` _ PAVICS-landing-add-pavics-homepage-notebooks-to-jenkins-testsuite/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb::Cell 1 _ Notebook cell execution failed Cell 1: Cell outputs differ Input: freq = 'YS' print(f'calculating yearly output using freq="{freq}"') with ProgressBar(): out = atmos.tx_days_above(tasmax=ds.tasmax, thresh='27 degC', freq=freq) # Yearly frequency # Average over spatial domain and plot time-series fig1 = plt.figure(figsize=(20,4)) plt.subplot(1,2,1) out.mean(dim=['lon','lat'], keep_attrs=True).plot() plt.title('spatial mean') # Calculate a 30 year climatology and plot a map plt.subplot(1,2,2) subset.subset_time(out, start_date='1981', end_date='2010').mean(dim='time', keep_attrs=True).plot() plt.title('1981-2010 mean') display() Traceback: mismatch 'stdout' assert reference_output == test_output failed: 'calculating ...one | 1.0s\n' == 'calculating ...one | 1.0s\n' Skipping 94 identical leading characters in diff, use -v to show - mpleted | 18.0s + mpleted | 1min 47.7s [########################################] | 100% Completed |100% Done | 1.0s ``` --- ...PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb index 8af502f..edc0728 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb @@ -834,6 +834,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "freq = 'YS'\n", "print(f'calculating yearly output using freq=\"{freq}\"')\n", "with ProgressBar():\n", From 87351c83ab3a87126408474363e02f0fd154a220 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 13:27:31 -0400 Subject: [PATCH 3/9] jenkins: ignore javascript output in notebook 5 This is of type `application/javascript` which should already be ignored by default (https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/blob/3806e67e7ad46c915797f483a373780c36f8a8f6/conftest.py#L3) Not sure why have to force ignore here but don't have time to investigate. --- .../PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb index 43d5fa2..b479611 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb @@ -16500,6 +16500,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "from xclim import ensembles as xens\n", "import pandas as pd\n", "from clisops.core import subset\n", From 9bf9a0c4278db6fc6a5a3668a68699cbdc13ddbc Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 15:29:50 -0400 Subject: [PATCH 4/9] jenkins: ignore output in notbook 1 For error: ``` _ PAVICS-landing-add-pavics-homepage-notebooks-to-jenkins-testsuite/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb::Cell 1 _ Notebook cell execution failed Cell 1: Cell outputs differ Input: import xarray as xr # This does not download the entire dataset, just the metadata and attributes describing the content. ds = xr.open_dataset(cds.access_urls["OPENDAP"], chunks='auto') # What we see here is an in-memory representation of the full content, the actual data is still on the server. ds Traceback: mismatch 'text/plain' assert reference_output == test_output failed: ' ? ^^^ ^ ^^^ + hunksize=(323, 320, 323), meta=np.ndarray> ? ^^ ^^ ^^^ - tasmax (time, lat, lon) float32 dask.array ? ^^^ ^ ^^^ + tasmax (time, lat, lon) float32 dask.array ? ^^ ^^ ^^^ - pr (time, lat, lon) float32 dask.array Date: Mon, 5 Jul 2021 15:31:55 -0400 Subject: [PATCH 5/9] jenkins: ignore output in notebook 3 For error: ``` _ PAVICS-landing-add-pavics-homepage-notebooks-to-jenkins-testsuite/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb::Cell 3 _ Notebook cell execution failed Cell 3: Cell outputs differ Input: with xclim.set_options(metadata_locales=['fr']): out_fr = atmos.tx_days_above(tasmax=ds.tasmax, thresh = '27 degC', freq='YS') out_fr Traceback: mismatch 'text/plain' assert reference_output == test_output failed: ' Date: Mon, 5 Jul 2021 15:36:51 -0400 Subject: [PATCH 6/9] jenkins: ignore 2 outputs for notebook 4 For errors: ``` _ PAVICS-landing-add-pavics-homepage-notebooks-to-jenkins-testsuite/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb::Cell 0 _ Notebook cell execution failed Cell 0: Cell outputs differ Input: from xclim import ensembles as xens from clisops.core import subset from pathlib import Path import matplotlib.pyplot as plt import warnings import logging logging.getLogger().disabled = True warnings.simplefilter("ignore") output = '/notebook_dir/writable-workspace/tmp/tutorial3/output' infolder = Path(output) # Create a list of rcp 4.5 files (n=11) ncfiles = [d for d in infolder.glob('tx_days_above*QS-DEC*rcp45*.nc')] #Create an ensemble dataset from the 11 simulations ds_ens = xens.create_ensemble(ncfiles) display(ds_ens) # Plot time series of single grid point lon = -66 lat = 48.5 ds1 = subset.subset_gridpoint(ds_ens, lon=lon, lat=lat) # plot summer season ds1.tx_days_above.sel(time=ds1['time.season']=='JJA').plot.line(figsize=(10,5), x='time' ,linewidth=0.5) #plt.title('RCP 4.5 individuals runs : ') display() Traceback: mismatch 'text/plain' assert reference_output == test_output failed: ' 27 degc - description: Seasonal number of days where daily maximum t... + Conventions: CF-1.5 + title: Ouranos standard ensemble of bias-adjusted cl... + history: MOHC pp to CMOR/NetCDF convertor (version 1.1... + institution: Ouranos Consortium on Regional Climatology an... + source: HadGEM2-CC (2011) atmosphere: HadGAM2(N96L60)... + driving_model: HadGEM2-CC ... ... - modeling_realm: atmos target_dataset: CANADA : ANUSPLIN interpolated Canada daily 3... target_dataset_references: CANADA : https://doi.org/10.1175/2011BAMS3132... - driving_institution: Norwegian Climate Centre ? ^^^^ ^ ^^^ ---- + driving_institution: Met Office Hadley Centre ? ^ ^^^^^ ++++ ^ + - driving_institute_id: NCC ? ^^ + driving_institute_id: MOHC ? ^^^ - crs: EPSG:4326 + crs: EPSG:4326 ? + + xclim_history: MOHC pp to CMOR/NetCDF convertor (version 1.1... ``` --- .../PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb index 557ab3d..3bcb8eb 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-4Ensembles.ipynb @@ -570,6 +570,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "from xclim import ensembles as xens\n", "from clisops.core import subset\n", "from pathlib import Path\n", @@ -1134,6 +1136,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "# Calculate percentiles\n", "ds_perc = xens.ensemble_percentiles(ds_ens, values=[10, 50, 90], split=False)\n", "display(ds_perc)\n", From 2633fe3f516ef40e3e59ae6ece49bfa40d746bc2 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 15:39:44 -0400 Subject: [PATCH 7/9] jenkins: ignore 2 outputs for notebook 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For errors: ``` _ PAVICS-landing-add-pavics-homepage-notebooks-to-jenkins-testsuite/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb::Cell 1 _ Notebook cell execution failed Cell 1: Cell outputs differ Input: # 30 year means and delta calculations window = 30 d30yAvg = ds_ens.rolling(year=window).mean() d30yAvg = d30yAvg.isel(year=slice(window-1,None)) # Select from the first full windowed mean # Select every horizons in 10 y intervals d30yAvg = d30yAvg.sel(year=(d30yAvg.year.values%10==0)) horizons = xr.DataArray([f'{yr - 29}-{yr}' for yr in d30yAvg.year.values], dims=dict(year=d30yAvg.year)) d30yAvg = d30yAvg.assign_coords(horizon=horizons) # Calculate deltas ref = d30yAvg.sel(year=(d30yAvg.horizon=='1981-2010')).squeeze() for v in d30yAvg.data_vars: with xr.set_options(keep_attrs=True): d30yAvg[f"{v}_delta"]= d30yAvg[v] - ref[v] for a in ['description', 'long_name']: d30yAvg[f"{v}_delta"].attrs[a] = f"{d30yAvg[f'{v}_delta'].attrs[a]} : delta vs 1981-2010" # Calculate percentiles on 30y normals d30yAvg_ens = xens.ensemble_percentiles(d30yAvg, split=False).load() display(d30yAvg_ens) map1 = d30yAvg_ens.tx_mean.hvplot.quadmesh(cmap='Spectral_r', geo=True,tiles='EsriImagery', framewise=False, frame_width=400) map1 Traceback: mismatch 'text/plain' assert reference_output == test_output failed: '
") return pn.Column(pn.Row(plt1.opts(legend_position='top_left'), rolling1)) ## Table summary @pn.depends(vars.param.value, regions.param.value, hors.param.value, delta.param.value) def create_table(v = vars.param.value, r=regions.param.value, h=hors.param.value, delta_flag=delta.param.value): title1 = f"Summary : {r} {h}" var_cols = var_cols = [vv for vv in df.columns if v in vv] if delta_flag: var_cols = [vv for vv in var_cols if 'delta' in vv] title1 = f"{title1} (delta vs 1981-2010)" else: var_cols = [vv for vv in var_cols if 'delta' not in vv] out = df[var_cols].iloc[(df.index.get_level_values('horizon') == h)&(df.index.get_level_values('region') == r)] return(out.sort_values(['season']).round(decimals=1).hvplot.table(title=title1, width=800, dynamic=True)) pn.Column(pn.pane.Markdown('# A simple PAVICS dashboard'), map1, regions1, plot_region_ts, create_table) Traceback: mismatch 'text/plain' assert reference_output == test_output failed: 'Column\n ...ion(function)' == 'Column\n ...ion(function)' Skipping 806 identical leading characters in diff, use -v to show - 41-2070', width=200) + 41-2070', value_throttled='2041-2070', width=200) [5] Column [0] Markdown(str) [1] Checkbox() [2] Column [0] Markdown(str) [1] Select(options=['Avignon', 'Bonaventure',...], value='Avignon') [3] ParamFunction(function) [4] ParamFunction(function) ``` --- .../PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb index b479611..2badf59 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-5Visualization.ipynb @@ -17208,6 +17208,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "# 30 year means and delta calculations\n", "window = 30\n", "d30yAvg = ds_ens.rolling(year=window).mean()\n", @@ -17665,6 +17667,8 @@ } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "\n", "rolling = pn.widgets.FloatInput(value=30, step=10, start=10, end=30, width=50)\n", "rolling1 = pn.Column(pn.pane.Markdown('Smoothing'),rolling)\n", "\n", From 9c03bf881d8eab136e29a5c85a65154c3668d5ba Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 16:33:43 -0400 Subject: [PATCH 8/9] jenkins: force use data from prod server for nb 1, 2, 3 They use too much .ncml which means too much .nc files to replicate to test servers. --- .../PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb | 2 +- .../PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb | 4 ++-- ...VICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb index 4a974bb..7a96d8b 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb @@ -43,7 +43,7 @@ ], "source": [ "from siphon.catalog import TDSCatalog\n", - "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n", + "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", "\n", "# Create Catalog\n", "cat = TDSCatalog(url)\n", diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb index 37247cb..aa98d3d 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-2Subsetting.ipynb @@ -922,7 +922,7 @@ "from IPython.display import display # Fancy representation of xarray objects\n", "import warnings\n", "warnings.simplefilter('ignore')\n", - "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n", + "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", "\n", "# Create Catalog\n", "cat = TDSCatalog(url)\n", @@ -3240,7 +3240,7 @@ } ], "source": [ - "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc'\n", + "testurl = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ouranos/climex/QC11d3_CCCma-CanESM2_rcp85/day/historical-r1-r1i1p1-rcp85/day_tasmin_historical-r1-r1i1p1-rcp85_kda_209904_se.nc' # TEST_USE_PROD_DATA\n", "ds = xr.open_dataset(testurl, chunks=dict(time=1))\n", "fig = plt.figure(figsize=(20,6))\n", "ax1 = plt.subplot(1,2,1)\n", diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb index 2c8daff..d4c98b2 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb @@ -777,7 +777,7 @@ "from dask.diagnostics import ProgressBar\n", "warnings.simplefilter('ignore')\n", "# TODO change address\n", - "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n", + "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", "\n", "# Create Catalog\n", "cat = TDSCatalog(url)\n", @@ -1638,7 +1638,7 @@ "from IPython.display import clear_output, display\n", "\n", "# cb-oura-1.0 thredds url\n", - "url = \"http://pavics.ouranos.ca:8083/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\"\n", + "url = \"http://pavics.ouranos.ca:8083/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", "\n", "# specify output folder for netcdf files\n", "output = '/notebook_dir/writable-workspace/tmp/tutorial3/output'\n", From d57556015f512c3753b4f2e38e211bf7c4390bf3 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Mon, 5 Jul 2021 16:45:18 -0400 Subject: [PATCH 9/9] jenkins: do not hit Thredds directly, avoid failing on test servers outside of Ouranos Since we use data from prod, test servers outside of Ouranos do not have acess to port 8083 so will fail. All prod Thredds access much go through port 443. --- ...PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb index d4c98b2..9df683b 100644 --- a/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb +++ b/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb @@ -1638,7 +1638,7 @@ "from IPython.display import clear_output, display\n", "\n", "# cb-oura-1.0 thredds url\n", - "url = \"http://pavics.ouranos.ca:8083/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", + "url = \"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip5/ouranos/cb-oura-1.0/catalog.xml\" # TEST_USE_PROD_DATA\n", "\n", "# specify output folder for netcdf files\n", "output = '/notebook_dir/writable-workspace/tmp/tutorial3/output'\n",