From 308187e8301d38e89daeed6a4b1eb706b3b579d2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 9 Nov 2023 14:55:49 -0500 Subject: [PATCH 1/4] adjust public WPS outputs sub-dir for weaver --- CHANGES.md | 12 +++++++++++- birdhouse/components/cowbird/default.env | 11 ++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5f0cd08a4..e8a2597c8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,17 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +- Fix `weaver` and `cowbird` inconsistencies for `public` WPS outputs directory handling. + + Because `cowbird` needs to mount multiple directories within the user-workspace for `jupyterhub`, it needs to define + a dedicated `public/wps_outputs` sub-directory to distinguish it from other `public` files not part of WPS outputs. + However, for WPS birds, other files than WPS outputs are irrelevant, and are therefore mounted directly in their + container. The variable `PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR` was being misused in the context of `weaver`, + causing WPS output URLs for `public` context to be nested as `/wpsoutputs/weaver/public/wps_outputs/{jobID}` + instead of the intended location `/wpsoutputs/weaver/public/{jobID}`, in contrast to user-context WPS outputs + located under `/wpsoutputs/weaver/users/{userID}/{jobID}`. + + Relates to [Ouranosinc/pavics-sdi#314](https://github.com/Ouranosinc/pavics-sdi/pull/314). [1.37.1](https://github.com/bird-house/birdhouse-deploy/tree/1.37.1) (2023-11-03) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env index 9f50d9f10..6d3ef197e 100644 --- a/birdhouse/components/cowbird/default.env +++ b/birdhouse/components/cowbird/default.env @@ -52,7 +52,15 @@ export USER_WORKSPACES="user_workspaces" # Subdirectory containing the hardlinks to the public WPS outputs data # This directory will be mounted on the JupyterLab instances and is located by default # in the ${USER_WORKSPACES} directory. -export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR="public/wps_outputs" +# NOTE: +# Most WPS birds do not have a concept of Public vs User-specific outputs. +# These birds will employ the same WPS output directory for all jobs, regardless of the user running it. +# By default, WPS output files will be stored under '${WPS_OUTPUTS_DIR}/', and must all be considered 'public'. +# Some WPS-capable birds such as Weaver do have a concept of Public/User-context for WPS outputs. +# In this case, files under '${WPS_OUTPUTS_DIR}/' should have an additional nesting +# with 'public' and 'users/{user_id}'. Variable 'PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR' will be shared for such cases. +export PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR=public +export COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR='${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR}/wps_outputs' # Default name for the secure-data-proxy service from Magpie. export SECURE_DATA_PROXY_NAME="secure-data-proxy" @@ -62,6 +70,7 @@ COWBIRD_MONGODB_DATA_DIR='${DATA_PERSIST_ROOT}/mongodb_cowbird_persist' DELAYED_EVAL=" $DELAYED_EVAL COWBIRD_MONGODB_DATA_DIR + COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR " # this dependency is only required if the mongo instance is the one provided in config/mongodb. From 09b356fb2df3e419c539d533b4a1868c98c74d0a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 9 Nov 2023 15:04:08 -0500 Subject: [PATCH 2/4] add missing variable fowarding for cowbird --- birdhouse/components/cowbird/config/cowbird/config.yml.template | 2 +- .../cowbird/config/jupyterhub/docker-compose-extra.yml | 2 +- birdhouse/components/cowbird/docker-compose-extra.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/birdhouse/components/cowbird/config/cowbird/config.yml.template b/birdhouse/components/cowbird/config/cowbird/config.yml.template index 2ea78e562..2b26195cc 100644 --- a/birdhouse/components/cowbird/config/cowbird/config.yml.template +++ b/birdhouse/components/cowbird/config/cowbird/config.yml.template @@ -35,7 +35,7 @@ handlers: wps_outputs_dir: ${WPS_OUTPUTS_DIR} secure_data_proxy_name: ${SECURE_DATA_PROXY_NAME} # wps_outputs_res_name: ${WPS_OUTPUTS_RES_NAME} - public_workspace_wps_outputs_subdir: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + public_workspace_wps_outputs_subdir: ${COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} # notebooks_dir_name: ${NOTEBOOKS_DIR_NAME} # user_wps_outputs_dir_name: ${USER_WPS_OUTPUTS_DIR_NAME} diff --git a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml index 6c98a641b..4565450e9 100644 --- a/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/config/jupyterhub/docker-compose-extra.yml @@ -14,6 +14,6 @@ services: jupyterhub: environment: WORKSPACE_DIR: ${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES} - PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} volumes: - "${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}:${DATA_PERSIST_SHARED_ROOT}/${USER_WORKSPACES}" diff --git a/birdhouse/components/cowbird/docker-compose-extra.yml b/birdhouse/components/cowbird/docker-compose-extra.yml index a6b28bd77..5ad767499 100644 --- a/birdhouse/components/cowbird/docker-compose-extra.yml +++ b/birdhouse/components/cowbird/docker-compose-extra.yml @@ -23,7 +23,7 @@ services: # root user COWBIRD_FILESYSTEM_ADMIN_UID: 0 COWBIRD_FILESYSTEM_ADMIN_GID: 0 - PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR: ${COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} SECURE_DATA_PROXY_NAME: ${SECURE_DATA_PROXY_NAME} # Note that WPS_OUTPUTS_DIR and WORKSPACE_DIR must both point to paths from the same volume. # This is to allow the creation of hardlinks between the wpsoutputs and the user workspace. From 6de51808bfa837ff06ffabec550a06d5f8bbd461 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 9 Nov 2023 15:44:57 -0500 Subject: [PATCH 3/4] add COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR to EXTRA_VARS --- birdhouse/components/cowbird/default.env | 1 + 1 file changed, 1 insertion(+) diff --git a/birdhouse/components/cowbird/default.env b/birdhouse/components/cowbird/default.env index 6d3ef197e..8100ddd84 100644 --- a/birdhouse/components/cowbird/default.env +++ b/birdhouse/components/cowbird/default.env @@ -18,6 +18,7 @@ EXTRA_VARS=' ${COWBIRD_LOG_LEVEL} ${USER_WORKSPACES} ${PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} + ${COWBIRD_PUBLIC_WORKSPACE_WPS_OUTPUTS_SUBDIR} ${SECURE_DATA_PROXY_NAME} ' # extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution From e783f87df1028dc29ceef940e39e6d81e7998c4c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 15:16:33 -0500 Subject: [PATCH 4/4] =?UTF-8?q?Bump=20version:=201.37.1=20=E2=86=92=201.37?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../config/canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 130975a4f..1cdaf7299 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.37.1 +current_version = 1.37.2 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2023-11-03T16:43:09Z +search = {current_version} 2023-11-10T20:16:33Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2023-11-03T16:43:09Z +values = 2023-11-10T20:16:33Z [bumpversion:file(version):birdhouse/config/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index e8a2597c8..92057c294 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[1.37.2](https://github.com/bird-house/birdhouse-deploy/tree/1.37.2) (2023-11-10) +------------------------------------------------------------------------------------------------------------------ + - Fix `weaver` and `cowbird` inconsistencies for `public` WPS outputs directory handling. Because `cowbird` needs to mount multiple directories within the user-workspace for `jupyterhub`, it needs to define diff --git a/Makefile b/Makefile index 2289f8b7f..f4ee81b00 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 1.37.1 +override APP_VERSION := 1.37.2 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index f406f4ff5..f6f57e9db 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ for a full-fledged production platform. * - releases - | |latest-version| |commits-since| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.37.1.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/1.37.2.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/1.37.1...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/1.37.2...master -.. |latest-version| image:: https://img.shields.io/badge/tag-1.37.1-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-1.37.2-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/1.37.1 + :target: https://github.com/bird-house/birdhouse-deploy/tree/1.37.2 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index 8bb917de1..f42af8c46 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -1.37.1 2023-11-03T16:43:09Z +1.37.2 2023-11-10T20:16:33Z diff --git a/birdhouse/config/canarie-api/docker_configuration.py.template b/birdhouse/config/canarie-api/docker_configuration.py.template index a457a3664..773fe82e1 100644 --- a/birdhouse/config/canarie-api/docker_configuration.py.template +++ b/birdhouse/config/canarie-api/docker_configuration.py.template @@ -109,8 +109,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.1', - 'releaseTime': '2023-11-03T16:43:09Z', + 'version': '1.37.2', + 'releaseTime': '2023-11-10T20:16:33Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', @@ -142,8 +142,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '1.37.1', - 'releaseTime': '2023-11-03T16:43:09Z', + 'version': '1.37.2', + 'releaseTime': '2023-11-10T20:16:33Z', 'institution': 'Ouranos', 'researchSubject': 'Climatology', 'supportEmail': '${SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index d33d91eea..c237710c6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '1.37.1' +version = '1.37.2' # The full version, including alpha/beta/rc tags. -release = '1.37.1' +release = '1.37.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.