diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..55337e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ + +.idea/ + +**/.DS_Store + +built_images.txt + +base_images/build-image-local.sh + +*.code-workspace \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5736a82 --- /dev/null +++ b/LICENSE @@ -0,0 +1,192 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright (c) 2022 California Institute of Technology (“Caltech”) U.S. Government sponsorship acknowledged, + and United States Government as represented by the Administrator of the National Aeronautics and Space Administration. + All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 2199b3b..a6fb253 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ # maap-workspaces Repository dedicated to building maap workspaces. + +## CI/CD setup +If using gitlab, create a blank project and copy the gitlab ci file to it. +Set up a webhook from this repository to trigger pipelines builds on the gitlab project. + +### CI Variables + +`FORCE_REF_BUILD`: variable used to force build a specific branch or commit hash. + +`BUILD_ALL_BASE_IMAGES`: Builds all images listed under the [base_images](/base_images) dir. + +`BUILD_SPECIFIC_BASE_IMAGES`: Comma separated list of base images to build. Names should match dir names under +[base_images](/base_images) dir + + +### Working of CI + +Use the above variables to control the execution of a CI pipline when manually triggering a pipline. +By default, when the CI receives a webhook, it will get the latest commit on the repo and list the files that have changed. + +If files changed match `base_images/*/*` it will trigger a build of those images. +Any other files changed currently does not trigger image builds. + +TODO: On files changed under jupyterlab dir, build all images. + +## Devfile publication + +To be listed in the ADE stack directory, the devfile and its metadata must be included in the [devfiles](/devfiles) directory. diff --git a/base_images/build-image.sh b/base_images/build-image.sh new file mode 100755 index 0000000..16b1bdd --- /dev/null +++ b/base_images/build-image.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ex +base_image_dir=$(dirname $0) +# Check if on a branch or in a detached HEAD state get commit sha +BRANCH=$(basename $(git symbolic-ref -q --short HEAD || git rev-parse --short HEAD)) +DIRS="vanilla r rgedi rsgislib" +if [[ ! -z "$@" ]]; then + DIRS=$@ +fi +for dir in ${DIRS}; do + pushd $base_image_dir/$dir + IMAGE_NAME=$(basename $dir) + IMAGE_REF=${CI_REGISTRY_IMAGE}/base_images/${IMAGE_NAME}:${BRANCH} + docker build -t ${IMAGE_REF} --build-arg IMAGE_REF=${IMAGE_REF} -f docker/Dockerfile . + docker push ${IMAGE_REF} + popd + echo "$IMAGE_REF" >> built_images.txt +done + diff --git a/base_images/isce2/docker/Dockerfile b/base_images/isce2/docker/Dockerfile new file mode 100644 index 0000000..f88a56a --- /dev/null +++ b/base_images/isce2/docker/Dockerfile @@ -0,0 +1,45 @@ +FROM continuumio/miniconda3:22.11.1 +ENV LANG en_US.UTF-8 +ENV TZ US/Pacific +ARG DEBIAN_FRONTEND=noninteractive + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN set -ex \ + && apt-get update \ + && apt-get install -y \ + libfftw3-3=3.3.8-2 \ + libgdal28=3.2.2+dfsg-2+deb11u2 \ + libhdf4-0=4.2.15-3 \ + libhdf5-103=1.10.6+repack-4+deb11u1 \ + libopencv-core4.5=4.5.1+dfsg-5 \ + libopencv-highgui4.5=4.5.1+dfsg-5 \ + libopencv-imgproc4.5=4.5.1+dfsg-5 \ + python3-gdal=3.2.2+dfsg-2+deb11u2 \ + python3-h5py=2.10.0-9 \ + python3-numpy=1:1.19.5-1 \ + python3-scipy=1.6.0-2 && \ + apt-get clean + + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +RUN conda install -y -c conda-forge mamba=1.3.1 + +# need to uninstall jupyter_server_terminals as it conflicts with Jupyterlab 3.4.x. Doesn't seem to break anything +# but can get rid of if/when we upgrade jlab +RUN mamba install -y -c conda-forge -c plant plant=0.1.89dev isce2=2.6.2 matplotlib=3.5.1 Cython=0.29.28 \ + numba=0.55.1 pygeos=0.14.0 pyproj=3.4.1 rasterio=1.3.6 && \ + pip uninstall -y jupyter_server_terminals && \ + find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} diff --git a/base_images/isce3/docker/Dockerfile b/base_images/isce3/docker/Dockerfile new file mode 100644 index 0000000..e054a86 --- /dev/null +++ b/base_images/isce3/docker/Dockerfile @@ -0,0 +1,30 @@ +FROM continuumio/miniconda3:22.11.1 +ENV LANG en_US.UTF-8 +ENV TZ US/Pacific +ARG DEBIAN_FRONTEND=noninteractive + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN set -ex \ + && apt-get update + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +RUN conda install -y -c conda-forge mamba=1.4.2 + +RUN mamba install -y -c conda-forge isce3=0.12.0 xarray=2023.4.2 \ + hvplot=0.8.3 fsspec=2023.5.0 scikit-learn=1.2.2 && \ + find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy + +RUN pip install git+https://github.com/opera-adt/RTC.git@v0.3 + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} diff --git a/base_images/pangeo/docker/Dockerfile b/base_images/pangeo/docker/Dockerfile new file mode 100644 index 0000000..32966cf --- /dev/null +++ b/base_images/pangeo/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM continuumio/miniconda3:22.11.1 +ENV LANG en_US.UTF-8 +ENV TZ US/Pacific +ARG DEBIAN_FRONTEND=noninteractive + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +RUN conda install -c conda-forge mamba && \ + mamba install -c conda-forge -y gdal=3.6.2 matplotlib=3.6.2 Cython=0.29.33 h5py=3.7.0 numba=0.56.4 \ + pygeos=0.14 pyproj=3.4.1 rasterio=1.3.4 scipy=1.10.0 \ + && find /opt/conda/ -follow -type f -name '*.a' -delete \ + && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && /opt/conda/bin/conda clean -afy + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} \ No newline at end of file diff --git a/base_images/r/docker/Dockerfile b/base_images/r/docker/Dockerfile new file mode 100644 index 0000000..e17d3f1 --- /dev/null +++ b/base_images/r/docker/Dockerfile @@ -0,0 +1,29 @@ +FROM continuumio/miniconda3:22.11.1 +ENV LANG en_US.UTF-8 +ENV TZ US/Pacific +ARG DEBIAN_FRONTEND=noninteractive + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN set -ex \ + && apt-get update + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +RUN conda install -y -c conda-forge mamba=1.3.1 + +RUN mamba install -y -c conda-forge r==4.2 r-rgdal==1.5_32 r-sf==1.0_7 r-irkernel==1.3.2 r-gridExtra==2.3 \ + r-tidyverse==2.0.0 r-randomForest==4.7_1.1 r-raster==3.6_20 r-data.table==1.14.8 r-rlist==0.4.6.2 \ + r-gdalutils==2.0.3.2 r-stringr==1.5.0 && \ + find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} diff --git a/base_images/rgedi/docker/Dockerfile b/base_images/rgedi/docker/Dockerfile new file mode 100644 index 0000000..603b0a0 --- /dev/null +++ b/base_images/rgedi/docker/Dockerfile @@ -0,0 +1,23 @@ +FROM continuumio/miniconda3:4.12.0 + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd && echo "source activate r-with-gdal" > ~/.bashrc +RUN apt-get update && apt-get install -y libxt-dev && apt-get clean + +RUN conda install -y -c conda-forge mamba && \ + mamba install -y -c conda-forge nb_conda_kernels +RUN mamba create --name r-with-gdal -c conda-forge -c r -c csdms-stack gdal r-rgdal r-sf r-irkernel r-gridExtra r-tidyverse \ + r-randomForest r-raster r-data.table r-rlist r-gdalutils r-stringr r-devtools sysroot_linux-64=2.17 gcc r-lwgeom szip --yes && \ + /opt/conda/bin/conda clean -afy +RUN mkdir -p ~/.R/ && echo "LDFLAGS=-lproj" >> ~/.R/Makevars && \ + conda run --no-capture-output -n r-with-gdal Rscript -e "devtools::install_git('https://github.com/carlos-alberto-silva/rGEDI', dependencies=TRUE)" + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} diff --git a/base_images/vanilla/docker/Dockerfile b/base_images/vanilla/docker/Dockerfile new file mode 100644 index 0000000..77c23bb --- /dev/null +++ b/base_images/vanilla/docker/Dockerfile @@ -0,0 +1,21 @@ +FROM continuumio/miniconda3:22.11.1 + +# install maap-py library +ENV MAAP_CONF='/maap-py/' +RUN git clone --single-branch --branch master https://github.com/MAAP-Project/maap-py.git \ + && cd maap-py \ + && pip install -e . + +RUN mkdir /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +RUN conda install -c conda-forge mamba && \ + mamba install -c conda-forge -y gdal=3.4.1 matplotlib=3.5.1 Cython=0.29.28 h5py=3.6.0 numba=0.55.1 \ + pygeos=0.12.0 pyproj=3.3.0 rasterio=1.2.10 scipy=1.8.0 && \ + find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy + +ARG IMAGE_REF +ENV DOCKERIMAGE_PATH=${IMAGE_REF} diff --git a/devfiles/.htaccess b/devfiles/.htaccess new file mode 100644 index 0000000..9c28593 --- /dev/null +++ b/devfiles/.htaccess @@ -0,0 +1 @@ +DirectoryIndex index.json \ No newline at end of file diff --git a/devfiles/edav/devfile/devfile.yaml b/devfiles/edav/devfile/devfile.yaml new file mode 100644 index 0000000..89bedf3 --- /dev/null +++ b/devfiles/edav/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: edav- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/edav:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/edav/devfile/earth.png b/devfiles/edav/devfile/earth.png new file mode 100644 index 0000000..516a31b Binary files /dev/null and b/devfiles/edav/devfile/earth.png differ diff --git a/devfiles/edav/devfile/meta.yaml b/devfiles/edav/devfile/meta.yaml new file mode 100644 index 0000000..2977c47 --- /dev/null +++ b/devfiles/edav/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "MAAP ESA EDAV" +description: Latest version of MAAP ESA EDAV +tags: ["EDAV", "MAAP"] +icon: /devfiles/edav/devfile/earth.png +globalMemoryLimit: 2710Mi diff --git a/devfiles/index_example.json b/devfiles/index_example.json new file mode 100644 index 0000000..6747331 --- /dev/null +++ b/devfiles/index_example.json @@ -0,0 +1,88 @@ +[ + { + "displayName": "R", + "description": "MAAP RSGISLib workspace v2.0. Released September 2022.", + "tags": [ + "Python", + "R", + "JupyterLab", + "MAAP" + ], + "icon": "/images/r.png", + "globalMemoryLimit": "2710Mi", + "links": { + "self": "/devfiles/maap-r/devfile_2.0.yaml" + } + }, + { + "displayName": "Basic", + "description": "MAAP Basic workspace v2.0. Released September 2022.", + "tags": [ + "JupyterLab", + "Python", + "MAAP" + ], + "icon": "/images/jupyter.png", + "globalMemoryLimit": "2810Mi", + "links": { + "self": "/devfiles/maap-vanilla/devfile_2.0.yaml" + } + }, + { + "displayName": "MAAP RGEDI Stable", + "description": "Latest version of MAAP RGEDI", + "tags": [ + "JupyterLab", + "Python", + "R", + "MAAP" + ], + "icon": "/images/r.png", + "globalMemoryLimit": "2710Mi", + "links": { + "self": "/devfiles/rgedi/devfile.yaml" + } + }, + { + "displayName": "MAAP ESA EDAV", + "description": "Latest version of MAAP ESA EDAV", + "tags": [ + "EDAV", + "MAAP" + ], + "icon": "/images/earth.png", + "globalMemoryLimit": "2710Mi", + "links": { + "self": "/devfiles/edav/devfile.yaml" + } + }, + { + "displayName": "R (v1.0)", + "description": "MAAP RSGISLib workspace v1.0. Released February 2022.", + "tags": [ + "Python", + "R", + "JupyterLab", + "MAAP" + ], + "icon": "/images/r.png", + "globalMemoryLimit": "2710Mi", + "links": { + "self": "/devfiles/maap-r/devfile.yaml" + } + }, + { + "displayName": "Basic (v1.0)", + "description": "MAAP Basic workspace v1.0. Released February 2022.", + "tags": [ + "JupyterLab", + "Python", + "MAAP" + ], + "icon": "/images/jupyter.png", + "globalMemoryLimit": "2810Mi", + "links": { + "self": "/devfiles/maap-vanilla/devfile.yaml" + } + } +] diff --git a/devfiles/isce2/devfile/devfile.yaml b/devfiles/isce2/devfile/devfile.yaml new file mode 100644 index 0000000..06de73b --- /dev/null +++ b/devfiles/isce2/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: isce2- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/isce2:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/isce2/devfile/isce.png b/devfiles/isce2/devfile/isce.png new file mode 100644 index 0000000..a7f9859 Binary files /dev/null and b/devfiles/isce2/devfile/isce.png differ diff --git a/devfiles/isce2/devfile/meta.yaml b/devfiles/isce2/devfile/meta.yaml new file mode 100644 index 0000000..200158c --- /dev/null +++ b/devfiles/isce2/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "MAAP ISCE2/PLAnT" +description: Latest version of MAAP ISCE2/PLAnT +tags: ["JupyterLab", "Python", "MAAP", "ISCE2", "PLAnT"] +icon: /devfiles/isce2/devfile/isce.png +globalMemoryLimit: 2710Mi diff --git a/devfiles/isce3/devfile/devfile.yaml b/devfiles/isce3/devfile/devfile.yaml new file mode 100644 index 0000000..9c4b6ee --- /dev/null +++ b/devfiles/isce3/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: isce3- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/isce3:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/isce3/devfile/isce.png b/devfiles/isce3/devfile/isce.png new file mode 100644 index 0000000..a7f9859 Binary files /dev/null and b/devfiles/isce3/devfile/isce.png differ diff --git a/devfiles/isce3/devfile/meta.yaml b/devfiles/isce3/devfile/meta.yaml new file mode 100644 index 0000000..46ed876 --- /dev/null +++ b/devfiles/isce3/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "MAAP ISCE3" +description: Latest version of MAAP ISCE3 +tags: ["JupyterLab", "Python", "MAAP", "ISCE3"] +icon: /devfiles/isce3/devfile/isce.png +globalMemoryLimit: 2710Mi diff --git a/devfiles/pangeo/devfile/devfile.yaml b/devfiles/pangeo/devfile/devfile.yaml new file mode 100644 index 0000000..d256bec --- /dev/null +++ b/devfiles/pangeo/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: pangeo- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/pangeo:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/pangeo/devfile/meta.yaml b/devfiles/pangeo/devfile/meta.yaml new file mode 100644 index 0000000..6b4dd28 --- /dev/null +++ b/devfiles/pangeo/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "Pangeo" +description: "Version: 2023.04.15" +tags: ["Pangeo", "JupyterLab", "MAAP"] +icon: /devfiles/pangeo/devfile/pangeo_simple_logo.svg +globalMemoryLimit: 2710Mi diff --git a/devfiles/pangeo/devfile/pangeo_simple_logo.svg b/devfiles/pangeo/devfile/pangeo_simple_logo.svg new file mode 100644 index 0000000..7097d95 --- /dev/null +++ b/devfiles/pangeo/devfile/pangeo_simple_logo.svg @@ -0,0 +1,24 @@ + + + + + pangeo_simple_logo + + + + + + + + + + + + + + + diff --git a/devfiles/r/devfile/devfile.yaml b/devfiles/r/devfile/devfile.yaml new file mode 100644 index 0000000..e23e6ba --- /dev/null +++ b/devfiles/r/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: r- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/r:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/r/devfile/meta.yaml b/devfiles/r/devfile/meta.yaml new file mode 100644 index 0000000..0df23cd --- /dev/null +++ b/devfiles/r/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "MAAP R Stable" +description: Latest version of MAAP R +tags: ["Python", "R", "JupyterLab", "MAAP"] +icon: /devfiles/r/devfile/r.png +globalMemoryLimit: 2710Mi diff --git a/devfiles/r/devfile/r.png b/devfiles/r/devfile/r.png new file mode 100644 index 0000000..bcbd6df Binary files /dev/null and b/devfiles/r/devfile/r.png differ diff --git a/devfiles/rgedi/devfile/devfile.yaml b/devfiles/rgedi/devfile/devfile.yaml new file mode 100644 index 0000000..8f29b1d --- /dev/null +++ b/devfiles/rgedi/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: rgedi- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/rgedi:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/rgedi/devfile/meta.yaml b/devfiles/rgedi/devfile/meta.yaml new file mode 100644 index 0000000..a6adeaf --- /dev/null +++ b/devfiles/rgedi/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "MAAP RGEDI Stable" +description: Latest version of MAAP RGEDI +tags: ["Python", "R", "JupyterLab", "MAAP"] +icon: /devfiles/rgedi/devfile/r.png +globalMemoryLimit: 2710Mi diff --git a/devfiles/rgedi/devfile/r.png b/devfiles/rgedi/devfile/r.png new file mode 100644 index 0000000..bcbd6df Binary files /dev/null and b/devfiles/rgedi/devfile/r.png differ diff --git a/devfiles/vanilla/devfile/devfile.yaml b/devfiles/vanilla/devfile/devfile.yaml new file mode 100644 index 0000000..02280e2 --- /dev/null +++ b/devfiles/vanilla/devfile/devfile.yaml @@ -0,0 +1,95 @@ +apiVersion: 1.0.0 +metadata: + generateName: vanilla- +attributes: + editorFree: 'true' +components: + - endpoints: + - attributes: + type: ide + discoverable: 'false' + path: / + public: 'true' + protocol: http + name: jupyter + port: 3100 + referenceContent: | + kind: List + items: + - apiVersion: v1 + kind: Pod + metadata: + name: ws + labels: + ssh: enabled + spec: + volumes: + - name: ws-pvc + persistentVolumeClaim: + claimName: ws + - name: s3fs-volume + emptyDir: {} + containers: + - name: jupyter + image: 'mas.dit.maap-project.org/root/maap-workspaces/jupyterlab3/vanilla:develop' + imagePullPolicy: Always + resources: + limits: + memory: 8096Mi + volumeMounts: + - name: ws-pvc + mountPath: /projects + subPath: projects + - name: s3fs-volume + mountPath: /projects/.jupyter + subPath: dotjupyter + - name: s3fs-volume + mountPath: /projects/my-private-bucket + subPath: my-private-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/my-public-bucket + subPath: my-public-bucket + mountPropagation: HostToContainer + - name: s3fs-volume + mountPath: /projects/shared-buckets + subPath: shared-buckets + mountPropagation: HostToContainer + - name: s3fs + image: 'mas.ops.maap-project.org/root/che-sidecar-s3fs:master' + imagePullPolicy: Always + resources: + limits: + memory: 256Mi + securityContext: + privileged: true + volumeMounts: + - name: s3fs-volume + mountPath: /my-public-bucket + subPath: my-public-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /my-private-bucket + subPath: my-private-bucket + mountPropagation: Bidirectional + - name: s3fs-volume + mountPath: /shared-buckets + subPath: shared-buckets + mountPropagation: Bidirectional + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ws + spec: + storageClassName: nfs-client-che + - apiVersion: v1 + kind: Service + metadata: + generateName: sshport- + spec: + type: NodePort + ports: + - port: 22 + selector: + ssh: enabled + type: kubernetes diff --git a/devfiles/vanilla/devfile/jupyter.png b/devfiles/vanilla/devfile/jupyter.png new file mode 100644 index 0000000..2007cbb Binary files /dev/null and b/devfiles/vanilla/devfile/jupyter.png differ diff --git a/devfiles/vanilla/devfile/meta.yaml b/devfiles/vanilla/devfile/meta.yaml new file mode 100644 index 0000000..5ddab6f --- /dev/null +++ b/devfiles/vanilla/devfile/meta.yaml @@ -0,0 +1,6 @@ +--- +displayName: "Basic Stable" +description: Latest version of MAAP Basic +tags: ["JupyterLab", "Python", "MAAP"] +icon: /devfiles/vanilla/devfile/jupyter.png +globalMemoryLimit: 2710Mi diff --git a/edav/build-image.sh b/edav/build-image.sh new file mode 100755 index 0000000..a0bd5fc --- /dev/null +++ b/edav/build-image.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +edav_dir=$(dirname $0) +BRANCH=$(git name-rev --name-only HEAD) +BRANCH=$(basename ${BRANCH}) +pushd ${edav_dir} +IMAGE_REF=${CI_REGISTRY_IMAGE}/edav:${BRANCH} +docker build --no-cache -t ${IMAGE_REF} -f docker/Dockerfile . +docker push ${IMAGE_REF} diff --git a/edav/discovery.json b/edav/discovery.json new file mode 100644 index 0000000..d41e73b --- /dev/null +++ b/edav/discovery.json @@ -0,0 +1,593 @@ +[ + { + "id": "maap", + "name": "MAAP", + "opensearchUrl": "https://edav-das.val.esa-maap.org/opensearch", + "wcsUrl": "https://edav-das.val.esa-maap.org/wcs", + "internalWcsUrl": "http://edav-backend-mwcs.edav:680/wcs", + "staticCatalogue": true, + "additionalDatasetConfig": { + "ESACCI_Biomass_L4_AGB": { + "aoiRequired": true, + "fixedTime": true + }, + "FOS": { + "disabled": true + } + } + }, + { + "id": "nasa", + "name": "Nasa", + "opensearchUrl": "https://nasa-edav-das.val.esa-maap.org/opensearch", + "wcsUrl": "https://nasa-edav-das.val.esa-maap.org/wcs", + "internalWcsUrl": "http://edav-backend-mwcsnasa.edav:680/wcs", + "staticCatalogue": true, + "additionalDatasetConfig": { + "GEDI02_A": { + "type": "vector", + "fixedTime": false + }, + "GEDI02_B": { + "type": "vector", + "fixedTime": false + } + } + }, + { + "id": "maap_user", + "name": "User data", + "opensearchUrl": "https://edav-das-vap.val.esa-maap.org/opensearch", + "wcsUrl": "https://edav-das-vap.val.esa-maap.org/wcs", + "internalWcsUrl": "http://edav-backend-mwcsvap.edav:680/wcs" + }, + { + "id": "creodias", + "name": "Creodias", + "opensearchUrl": "https://maap-creodias.adamplatform.eu/opensearch/v1", + "wcsUrl": "https://maap-creodias.adamplatform.eu/wcs", + "staticCatalogue": true, + "additionalDatasetConfig": { + "FOS": { + "disabled": true + } + } + }, + { + "id": "aux", + "name": "External data", + "edavServices": { + "wcsServiceUrl": "https://edav-wcs.adamplatform.eu/wcs", + "cswServiceUrl": "https://edav-pycsw.adamplatform.eu/pycsw/csw.py", + "wpsServiceUrl": "https://edav-wps.adamplatform.eu/wps" + }, + "fosService": { + "url": "/fos", + "intendedUse": "view", + "applicationId": "maap", + "dasAccess": { + "opensearchUrl": "https://edav-das.val.esa-maap.org/opensearch", + "datasetId": "FOS" + } + }, + "datasets": [{ + "id": "fos", + "name": "FOS plots", + "datasetType": "fos", + "description": "The Forest Observation System is an international cooperation to establish a global in-situ forest biomass database to support earth observation and to encourage investment in relevant field-based observations and science. The Forest Observation System provides well curated biomass plot data in a unified format, that is aggregated from tree level data consistently across different networks." + }, { + "id": "biosar-1", + "name": "Biosar 1", + "datasetType": "edav", + "description": "ESA Biosar 1 campaign data", + "type": "raster", + "cswCollection": "biosar1_SLC", + "coverages": [{ + "id": "biosar_amplitude", + "name": "Amplitude", + "domain": { + "min": 0, + "max": 10, + "noData": -9999 + }, + "wcsCoverage": "biosar1_SLC", + "wcsSubset": { + "id": "gfix", + "value": "1", + "idx": 2 + } + }, { + "id": "biosar_slc_q", + "name": "Q", + "domain": { + "min": -5, + "max": 5, + "noData": -9999 + }, + "wcsCoverage": "biosar1_SLC", + "wcsSubset": { + "id": "gfix", + "value": "2", + "idx": 2 + } + }, { + "id": "biosar_slc_i", + "name": "I", + "domain": { + "min": -5, + "max": 5, + "noData": -9999 + }, + "wcsCoverage": "biosar1_SLC", + "wcsSubset": { + "id": "gfix", + "value": "3", + "idx": 2 + } + }], + "dimensions": [{ + "id": "scene", + "name": "Scene number", + "domain": { + "values": [{ + "value": 105 + }, { + "value": 106 + }, { + "value": 107 + }, { + "value": 109 + }, { + "value": 301 + }, { + "value": 302 + }, { + "value": 303 + }, { + "value": 304 + }, { + "value": 306 + }, { + "value": 406 + }, { + "value": 407 + }, { + "value": 408 + }, { + "value": 409 + }, { + "value": 411 + }, { + "value": 110 + }, { + "value": 206 + }, { + "value": 412 + }] + }, + "wcsSubset": { + "id": "gfix", + "idx": 0 + }, + "wcsResponseKey": "gfix", + "tarFilenameRegex": "_([0-9]*)_" + }, { + "id": "polarization", + "name": "Polarization", + "domain": { + "values": [{ + "value": "11", + "label": "VV" + }, { + "value": "00", + "label": "HH" + }, { + "value": "10", + "label": "VH" + }, { + "value": "01", + "label": "HV" + }] + }, + "wcsSubset": { + "id": "gfix", + "idx": 1 + }, + "wcsResponseKey": "gfix", + "tarFilenameRegex": "SLC_([A-Z]*)_" + }] + },{ + "id": "globbiomass", + "name": "Globbiomass", + "datasetType": "edav", + "description": "The data products consist of global datasets including estimates of growing stock volume (GSV, unit: m3/ha) for the year 2010 and above ground biomass (AGB, unit: tons/ha i.e., Mg/ha) for the year 2010", + "type": "raster", + "cswCollection": "GLOBBIOMASS_AGB", + "coverages": [{ + "id": "agb", + "name": "Above ground biomass", + "units": "Mg · ha⁻¹", + "domain": { + "min": 0, + "max": 400, + "noData": -9999 + }, + "wcsCoverage": "GLOBBIOMASS_AGB_4326_0000889" + }, { + "id": "agb_err", + "name": "Above ground biomass error", + "units": "Mg · ha⁻¹", + "domain": { + "min": 0, + "max": 200, + "noData": -9999 + }, + "wcsCoverage": "GLOBBIOMASS_AGB_ERR_4326_0000889" + }, { + "id": "gsv", + "name": "Growing stock volume", + "units": "m³ · ha⁻¹", + "domain": { + "min": 0, + "max": 400, + "noData": -9999 + }, + "wcsCoverage": "GLOBBIOMASS_GSV_4326_0000889" + }, { + "id": "gsv_err", + "name": "Growing stock volume error", + "units": "m³ · ha⁻¹", + "domain": { + "min": 0, + "max": 200, + "noData": -9999 + }, + "wcsCoverage": "GLOBBIOMASS_GSV_ERR_4326_0000889" + }] + }, { + "id": "biomass_tomo_onera_lope", + "name": "Tomographic biomass Onera Lope", + "datasetType": "edav", + "description": "Synthetic tomographic biomass product", + "type": "volume", + "cswCollection": "TOMO_GROUND_ONERA", + "coverages": [{ + "id": "biomass", + "name": "Biomass", + "domain": { + "min": 0, + "max": 20, + "noData": -9999 + }, + "units": "g/m²", + "wcsCoverage": "TOMO_GROUND_ONERA_Lope_32732_48" + }], + "dimensions": [{ + "id": "height", + "name": "Height", + "domain": { + "min": -20, + "max": 70, + "step": 1 + }, + "units": "m", + "wcsSubset": { + "id": "h" + }, + "wcsResponseKey": "high" + }] + }, { + "id": "biomass_tomo_onera_lope_geo", + "name": "Tomographic biomass Onera Lope (GEO)", + "datasetType": "edav", + "type": "volume", + "cswCollection": "TOMO_GROUND_ONERA", + "coverages": [{ + "id": "biomass", + "name": "Biomass", + "domain": { + "min": 0, + "max": 20, + "noData": -9999 + }, + "units": "g/m²", + "wcsCoverage": "TOMO_GROUND_ONERA_Lope_GEO_32732_5" + }], + "dimensions": [{ + "id": "height", + "name": "Height", + "domain": { + "min": -20, + "max": 70, + "step": 1 + }, + "units": "m", + "wcsSubset": { + "id": "h" + }, + "wcsResponseKey": "high" + }] + }, { + "id": "congo_gedi_icesat", + "name": "GEDI Congo derived height", + "datasetType": "edav", + "type": "raster", + "coverages": [{ + "id": "height", + "name": "Mean Height", + "units": "m", + "domain": { + "min": 0, + "max": 150, + "noData": -9999 + }, + "wcsCoverage": "GEDI_icesat", + "wcsSubset": { + "id": "band", + "value": "1" + } + }, { + "id": "height_variance", + "name": "Height variance", + "units": "m", + "domain": { + "min": 0, + "max": 2, + "noData": -9999 + }, + "wcsCoverage": "GEDI_icesat", + "wcsSubset": { + "id": "band", + "value": "2" + } + }, { + "id": "num_shots", + "name": "Number of shots", + "domain": { + "min": 0, + "max": 100, + "noData": -9999 + }, + "wcsCoverage": "GEDI_icesat", + "wcsSubset": { + "id": "band", + "value": "3" + } + }] + }, { + "id": "gedi_rh", + "name": "GEDI L2A RH", + "datasetType": "edav", + "type": "vertical_profile", + "coverages": [{ + "id": "rh", + "name": "Relative height", + "description": "Relative height metrics at 1 % interval", + "units": "m", + "domain": { + "min": -10, + "max": 70, + "noData": -9999 + }, + "wcsCoverage": "GEDI02_A_RH", + "default": { + "range": { + "min": 0, + "max": 50 + }, + "colorScale": "turbo" + } + }], + "fixedTime": "2020-10-27T22:49:57Z", + "dimensions": [{ + "id": "beam", + "name": "Beam", + "domain": { + "values": [{ + "value": "0", + "label": "0" + }, { + "value": "1", + "label": "1" + }, { + "value": "10", + "label": "10" + }, { + "value": "11", + "label": "11" + }, { + "value": "101", + "label": "101" + }, { + "value": "110", + "label": "110" + }, { + "value": "1000", + "label": "1000" + }, { + "value": "1011", + "label": "1011" + }] + }, + "allowRange": true, + "wcsSubset": { + "id": "gfix", + "idx": 0 + } + }], + "verticalScaleConfig": { + "min": 0.2, + "max": 2, + "step": 0.05, + "default": 0.5 + } + }, { + "id": "gedi_l4a", + "name": "GEDI L4B", + "datasetType": "edav", + "type": "raster", + "coverages": [{ + "id": "mu", + "name": "Mean aboveground biomass (MU)", + "units": "Mg · ha⁻¹", + "description": "Mean aboveground biomass density: Estimated mean AGBD for the 1 km grid cell, including forest and non-forest", + "domain": { + "min": 0, + "max": 600, + "noData": -9999 + }, + "wcsCoverage": "GEDI04_B_MU_4326_001", + "default": { + "range": { + "min": 0, + "max": 350 + }, + "colorScale": "viridis" + } + }, { + "id": "V1", + "name": "Variance component 1 (V1)", + "description": "Uncertainty in the estimate of mean biomass due to the field-to-GEDI model used in L4A", + "domain": { + "min": 0, + "max": 1000, + "noData": -9999 + }, + "wcsCoverage": "GEDI04_B_V1_4326_001", + "default": { + "range": { + "min": 0, + "max": 100 + }, + "colorScale": "turbo" + } + }, { + "id": "V2", + "name": "Variance component 2 (V2)", + "description": "If Mode of Inference = 1, this is the uncertainty due to GEDI's sampling of the 1 km cell. If Mode of Inference = 2, this is uncertainty owing to the model predicting biomass using wall-to-wall data, calibrated with the L4A footprint product", + "domain": { + "min": 0, + "max": 10000, + "noData": -9999 + }, + "wcsCoverage": "GEDI04_B_V2_4326_001", + "default": { + "range": { + "min": 0, + "max": 1000 + }, + "colorScale": "turbo" + } + }, { + "id": "SE", + "name": "Mean AGBD standard error (SE)", + "description": "Mean aboveground biomass density standard error: Standard Error of the mean estimate, combining sampling and modeling uncertainty", + "units": "Mg · ha⁻¹", + "domain": { + "min": 0, + "max": 500, + "noData": -9999 + }, + "wcsCoverage": "GEDI04_B_SE_4326_001", + "default": { + "range": { + "min": 0, + "max": 200 + }, + "colorScale": "turbo" + } + }, { + "id": "PE", + "name": "Mean AGB standard error % (PE)", + "description": "Standard error as a fraction of the estimated mean AGBD (PE). If >100%, the cell values are truncated to 100.", + "units": "%", + "domain": { + "min": 0, + "max": 100, + "noData": 255 + }, + "wcsCoverage": "GEDI04_B_PE_4326_001", + "default": { + "range": { + "min": 0, + "max": 100 + }, + "colorScale": "turbo" + } + }, { + "id": "NC", + "name": "Number of clusters (NC)", + "description": "Number of unique GEDI ground tracks with at least one high-quality waveform intersecting the grid cell", + "domain": { + "min": 0, + "max": 100 + }, + "wcsCoverage": "GEDI04_B_NC_4326_001", + "default": { + "range": { + "min": 0, + "max": 20 + }, + "colorScale": "turbo" + } + }, { + "id": "NS", + "name": "Number of samples (NS)", + "description": "Total number of high-quality waveforms across all ground tracks within the grid cell", + "domain": { + "min": 0, + "max": 1000 + }, + "wcsCoverage": "GEDI04_B_NS_4326_001", + "default": { + "range": { + "min": 0, + "max": 200 + }, + "colorScale": "turbo" + } + }, { + "id": "QF", + "name": "Quality flag (QF)", + "description": "0=Outside the GEDI domain, 1=Land surface, 2=Land surface and meets GEDI mission L1 requirement (Percent standard error <20% or Standard Error < 20 Mg ha-1)", + "domain": { + "min": 0, + "max": 2, + "step": 1 + }, + "wcsCoverage": "GEDI04_B_QF_4326_001", + "default": { + "colorScale": "jet" + } + }, { + "id": "PS", + "name": "Prediction stratum (PS)", + "description": "Determined by plant functional type and continent. PS is associated with an L4A model parameter covariance matrix that contributes to the Model Error Variance.", + "domain": { + "min": 0, + "max": 35, + "noData": 241, + "step": 1 + }, + "wcsCoverage": "GEDI04_B_PS_4326_001", + "default": { + "colorScale": "jet", + "clamp": false + } + }, { + "id": "MI", + "name": "Mode of interference (MI)", + "description": "Method used for a particular cell. Until mission completion, only those cells where hybrid inference is possible will be populated with a mean biomass value. 0=None applied, 1=Hybrid Model-Based, 2=Generalized Hierarchical Model-Based", + "domain": { + "min": 0, + "max": 2, + "step": 1 + }, + "default": { + "colorScale": "jet" + }, + "wcsCoverage": "GEDI04_B_MI_4326_001" + }] + }] + } +] \ No newline at end of file diff --git a/edav/docker/Dockerfile b/edav/docker/Dockerfile new file mode 100644 index 0000000..aca86a0 --- /dev/null +++ b/edav/docker/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:20.04 +RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git npm +#RUN git clone https://github.com/MAAP-Project/maap-edav-esa-front.git +RUN git clone https://github.com/grallewellyn/maap-edav-esa-front2.git +WORKDIR "/maap-edav-esa-front2" +RUN npm install && npm run build +COPY discovery.json /maap-edav-esa-front/dist/data/ + +FROM nginx:1.23.1 +RUN apt-get update && apt-get install -y python3 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +COPY --from=0 /maap-edav-esa-front/dist/ /usr/share/nginx/html + +COPY entrypoint.sh / +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] +# below is copied from nginx's Dockerfile +EXPOSE 3100 + +STOPSIGNAL SIGQUIT + +CMD ["nginx", "-g", "daemon off;"] diff --git a/edav/entrypoint.sh b/edav/entrypoint.sh new file mode 100755 index 0000000..b1d5e1e --- /dev/null +++ b/edav/entrypoint.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# A more robust method of constructing the workspace url prefix +get_workspace_url_prefix() { + python3 - <-che" + if test -z "$PREVIEW_URL" + then + PREVIEW_URL=$(get_workspace_url_prefix "$CHE_WORKSPACE_NAMESPACE-$CHE_WORKSPACE_ID") # Che 7 UAT configuration fallback where the default namespace is "-", this will be deprecated + fi + if test -z "$PREVIEW_URL" + then + PREVIEW_URL=$(get_workspace_url_prefix "$CHE_WORKSPACE_ID") # Che 6 configuration fallback where the default namespace is the workspace id, this will be deprecated + fi + if ! test -z "$PREVIEW_URL" # exit loop when it has a preview_url + then + break + fi +done +# end more robust method + +perl -pi -e "s|base href=\"|base href=\"$PREVIEW_URL|g" /usr/share/nginx/html/index.html +perl -pi -e "s|80;|3100;|g" /etc/nginx/conf.d/default.conf + +. docker-entrypoint.sh + diff --git a/gitlab-ci/.gitlab-ci.yml b/gitlab-ci/.gitlab-ci.yml new file mode 100644 index 0000000..cf0b9b2 --- /dev/null +++ b/gitlab-ci/.gitlab-ci.yml @@ -0,0 +1,23 @@ +stages: + - build + - setup + - trigger + - test + +generate-config: + stage: setup + script: + - bash ./generate_configs.sh + artifacts: + paths: + - stages.yml + only: + refs: + - main + +build_image_trigger: + stage: trigger + trigger: + include: + - artifact: stages.yml + job: generate-config diff --git a/gitlab-ci/README.md b/gitlab-ci/README.md new file mode 100644 index 0000000..a6fb253 --- /dev/null +++ b/gitlab-ci/README.md @@ -0,0 +1,30 @@ +# maap-workspaces +Repository dedicated to building maap workspaces. + +## CI/CD setup +If using gitlab, create a blank project and copy the gitlab ci file to it. +Set up a webhook from this repository to trigger pipelines builds on the gitlab project. + +### CI Variables + +`FORCE_REF_BUILD`: variable used to force build a specific branch or commit hash. + +`BUILD_ALL_BASE_IMAGES`: Builds all images listed under the [base_images](/base_images) dir. + +`BUILD_SPECIFIC_BASE_IMAGES`: Comma separated list of base images to build. Names should match dir names under +[base_images](/base_images) dir + + +### Working of CI + +Use the above variables to control the execution of a CI pipline when manually triggering a pipline. +By default, when the CI receives a webhook, it will get the latest commit on the repo and list the files that have changed. + +If files changed match `base_images/*/*` it will trigger a build of those images. +Any other files changed currently does not trigger image builds. + +TODO: On files changed under jupyterlab dir, build all images. + +## Devfile publication + +To be listed in the ADE stack directory, the devfile and its metadata must be included in the [devfiles](/devfiles) directory. diff --git a/gitlab-ci/generate_configs.sh b/gitlab-ci/generate_configs.sh new file mode 100644 index 0000000..7aa6b30 --- /dev/null +++ b/gitlab-ci/generate_configs.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -e +basedir=$( cd "$(dirname "$0")" ; pwd -P ) +git clone https://github.com/MAAP-Project/maap-workspaces.git +pushd maap-workspaces +LATEST_COMMIT=$(git log -n 1 --all --format='%h') +if [[ ! -z ${FORCE_REF_BUILD} ]]; then + LATEST_COMMIT=${FORCE_REF_BUILD} +fi +git checkout ${LATEST_COMMIT} + +if [[ ! -z ${BUILD_ALL_BASE_IMAGES} ]]; then + ls -d base_images/*/* > ${basedir}/files_changed.txt +elif [[ ! -z ${BUILD_SPECIFIC_BASE_IMAGES} ]]; then + base_image_array=(${BUILD_SPECIFIC_BASE_IMAGES}) + for path in ${base_image_array[@]}; do + echo "base_images/${path}/docker" >> ${basedir}/files_changed.txt + done +else + # Find files changed in the latest commit + echo "BUILD_ALL_BASE_IMAGES and BUILD_SPECIFIC_BASE_IMAGES unset" + echo "Getting files changed in last commit" + git diff --name-only HEAD HEAD~1 > ${basedir}/files_changed.txt +fi +cat ${basedir}/files_changed.txt +export LATEST_COMMIT +popd +rm -rf maap-workspaces +template="${basedir}/stage.yml.tmpl" +# For each file changed, check if its in one of the base images directory +cat ${basedir}/files_changed.txt | while read path +do + if [[ "$path" == base_images/*/* ]]; then + second_dir=$(echo "$path" | cut -d'/' -f2) + export BASE_IMAGE_TYPE="$second_dir" + echo "Adding stage for $path" + # Add the base image changed file to build stage for downstream pipeline + cat ${template} | CI_JOB_TOKEN='$CI_JOB_TOKEN' CI_REGISTRY='$CI_REGISTRY' envsubst >> stages.yml + else + echo "Path does not begin with base_images or does not have a second directory, will not do anything" + touch stages.yml + fi +done + +echo "Generate stages.yaml" +cat stages.yml || true diff --git a/gitlab-ci/stage.yml.tmpl b/gitlab-ci/stage.yml.tmpl new file mode 100644 index 0000000..3bbf66c --- /dev/null +++ b/gitlab-ci/stage.yml.tmpl @@ -0,0 +1,10 @@ +${BASE_IMAGE_TYPE}: + script: + - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + - git clone https://github.com/MAAP-Project/maap-workspaces.git + - pushd maap-workspaces + - git checkout ${LATEST_COMMIT} + - bash base_images/build-image.sh ${BASE_IMAGE_TYPE} + - cat built_images.txt + - export BASE_IMAGE_NAME=$(tail -1 built_images.txt) + - bash jupyterlab3/build-image.sh diff --git a/index.sh b/index.sh new file mode 100755 index 0000000..020d554 --- /dev/null +++ b/index.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +readarray -d '' metas < <(find devfiles -name 'meta.yaml' -print0) +N=0 + +echo [ + +for meta in "${metas[@]}"; do + if [ $N -gt 0 ] + then + echo , + fi + N=$((N+1)) + + META_DIR=$(dirname "${meta}") + echo -e "links:\n self: /${META_DIR}/devfile.yaml" >> "${meta}" + yq r --prettyPrint -j ${META_DIR}/meta.yaml +done + +echo ] diff --git a/jupyterlab3/build-image.sh b/jupyterlab3/build-image.sh new file mode 100644 index 0000000..ad5882b --- /dev/null +++ b/jupyterlab3/build-image.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -ex +jupyterlab_dir=$(dirname $0) +BRANCH=$(git name-rev --name-only HEAD) +BRANCH=$(basename ${BRANCH}) +pushd ${jupyterlab_dir} +if [[ -z ${BASE_IMAGE_NAME} ]]; then + echo "WARNING: No value provided for BASE_IMAGE_NAME, will ./build with default miniconda3 image" + BASE_IMAGE_NAME=${CI_REGISTRY_IMAGE}/base_images/vanilla:${BRANCH} +fi +[[ $(basename ${BASE_IMAGE_NAME}) =~ (([^:]*)\:) ]]; +BASE_IMAGE_TYPE=${BASH_REMATCH[2]} +IMAGE_REF=${CI_REGISTRY_IMAGE}/jupyterlab3/$(basename ${BASE_IMAGE_NAME}) +docker build --no-cache -t ${IMAGE_REF} --build-arg BASE_IMAGE_TYPE=${BASE_IMAGE_TYPE} --build-arg BASE_IMAGE=${BASE_IMAGE_NAME} -f docker/Dockerfile . +docker push ${IMAGE_REF} diff --git a/jupyterlab3/docker/Dockerfile b/jupyterlab3/docker/Dockerfile new file mode 100644 index 0000000..5d211f8 --- /dev/null +++ b/jupyterlab3/docker/Dockerfile @@ -0,0 +1,85 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} as jupyterlab_base + +ARG BASE_IMAGE_TYPE + +ADD . / + +RUN mkdir -p /projects +WORKDIR /projects +RUN sed -i -e 's/\/root/\/projects/g' /etc/passwd + +USER root + +# Install additional libraries required by Python packages which are in +# the minimal base image. Also install 'rsync' so the 'oc rsync' command +# can be used to copy files into the running container. +RUN apt-get clean && apt-get update && \ + apt-get install -y --no-install-recommends rsync gcc python3 vim && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install workspace specific packages for conda environment +RUN echo "Checking if environment.yml exists for ${BASE_IMAGE_TYPE}" \ + ; if [ -f "/${BASE_IMAGE_TYPE}/environment.yml" ]; then \ + echo "Installing packages from '/${BASE_IMAGE_TYPE}/environment.yml'" \ + ; mamba env update --name base --file "/${BASE_IMAGE_TYPE}/environment.yml" \ + ; else \ + echo "Installing packages from '/shared/environment.yml'" \ + ; mamba env update --name base --file "/shared/environment.yml" \ + ; fi + +RUN npm install typescript -g +RUN pip install jupyter-server==2.5.0 + +# Adjust permissions on home directory so writable by group root. +RUN chgrp -Rf root /home/$NB_USER && chmod -Rf g+w /home/$NB_USER + +# Grant access to jupyterlab config files for base url rewriting +RUN chmod a+rwx -R /opt/conda/lib/python*/site-packages/ + +# Adjust permissions on /etc/passwd so writable by group root. +RUN chmod g+w /etc/passwd + +############################### +# Non Custom Jupyter Extensions +############################### +RUN jupyter labextension install --no-build jupyterlab-plotly@5.5.0 +RUN jupyter labextension disable @jupyterlab/apputils-extension:announcements + +############################### +# Custom Jupyter Extensions +############################### +RUN jupyter labextension install @maap-jupyterlab/dps-jupyter-extension@0.2.0 --no-build + +# PyPi package prepended with 'maap' so it more easily discoverable +RUN pip install maap-jupyter-server-extension==1.0.0 +RUN jupyter server extension enable jupyter_server_extension + +RUN jupyter labextension install @maap-jupyterlab/umf-jupyter-extension@1.0.0 --no-build +RUN jupyter labextension install @maap-jupyterlab/maap-libs-jupyter-extension@1.0.0 --no-build +# RUN jupyter labextension install @maap-jupyterlab/edsc-jupyter-extension@latest --no-build +RUN jupyter labextension install @maap-jupyterlab/user-workspace-management-jupyter-extension@0.0.3 --no-build +RUN jupyter labextension install @maap-jupyterlab/maap-help-jupyter-extension@0.0.46 --no-build + +RUN jupyter lab build && \ + jupyter lab clean && \ + jlpm cache clean && \ + npm cache clean --force && \ + rm -rf $HOME/.node-gyp && \ + rm -rf $HOME/.local + +RUN find /opt/conda/ -follow -type f -name '*.a' -delete && \ + find /opt/conda/ -follow -type f -name '*.js.map' -delete && \ + /opt/conda/bin/conda clean -afy && \ + npm cache clean --force + +COPY ./entrypoint.sh /entrypoint.sh +COPY ./jupyter_patch /opt/jupyter_patch + +ENV ENVIRONMENTS_FILE_PATH=/opt/environments.json +COPY ./environments.json $ENVIRONMENTS_FILE_PATH + +RUN echo 'printf "\e[101mNOTE: This terminal uses the default environment which might not be the same as your notebook.\e[49m\n\n"' >> /etc/profile + +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] diff --git a/jupyterlab3/entrypoint.sh b/jupyterlab3/entrypoint.sh new file mode 100644 index 0000000..e04c173 --- /dev/null +++ b/jupyterlab3/entrypoint.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +# A more robust method of constructing the workspace url prefix +get_workspace_url_prefix() { + python3 - <-che" + if test -z "$PREVIEW_URL" + then + PREVIEW_URL=$(get_workspace_url_prefix "$CHE_WORKSPACE_NAMESPACE-$CHE_WORKSPACE_ID") # Che 7 UAT configuration fallback where the default namespace is "-", this will be deprecated + fi + if test -z "$PREVIEW_URL" + then + PREVIEW_URL=$(get_workspace_url_prefix "$CHE_WORKSPACE_ID") # Che 6 configuration fallback where the default namespace is the workspace id, this will be deprecated + fi + if ! test -z "$PREVIEW_URL" # exit loop when it has a preview_url + then + break + fi +done +# end more robust method + +# Fix Jupyterlab for Che in `single-host` mode. In `single-host` mode, Che uses URL path prefixes +# to distinguish workspaces. So for example, `https:///work/space/path/`. +# Because of this, we need to set Jupyter's `base_url` to `/work/space/path` so that all hrefs +# and links to files have the correct path prefix. HOWEVER, when the browser accesses those paths, +# the ingress/nginx proxy strips out the `base_url`! Even if the browser makes a request to `/work/space/path/lab`, +# Jupyter's web server, Tornado, only see a request for `/lab`. Tornado routes calls to the correct handler by +# matching the path against a known regular expression. Because `base_url` is configured to `/work/space/path`, +# Tornado only handles requests that have that path prefix, causing calls such as `/lab` to fail. The fix below +# allows `base_url` to be set so that HTTP *responses* include the `base_url` so that browsers make the correct +# call. However, it strips out `base_url` when listening for *requests* so that handles the ingress/nginx proxy +# requests correctly. +export NOTEBOOKLIBPATH=$(find /opt/conda/lib/ -maxdepth 3 -type d -name "notebook") +export JUPYTERSERVERLIBPATH=$(find /opt/conda/lib -maxdepth 3 -type d -name "jupyter_server") + +read -r -d '' JUPYTER_PATCH << EOM + # Fix for Tornado's inability to handle proxy requests + from tornado.routing import _RuleList + def fix_handlers(self, handlers: _RuleList, base_url: str): + for i in range(len(handlers)): + l = list(handlers[i]) + l[0] = l[0].replace(base_url.rstrip('/'), '') + handlers[i] = tuple(l) + return handlers + + def add_handlers(self, host_pattern: str, host_handlers: _RuleList) -> None: + super().add_handlers(host_pattern, self.fix_handlers(host_handlers, self.settings['base_url'])) +EOM + +if [[ -f "$JUPYTERSERVERLIBPATH/serverapp.py" ]]; then + perl -pi -e "s|(.*)\(web.Application\):|\$1\(web.Application\):\n$JUPYTER_PATCH|g" "$JUPYTERSERVERLIBPATH/serverapp.py" + perl -pi -e 's|(.*)__init__\(handlers(.*)|$1__init__\(self.fix_handlers\(handlers, base_url\)$2|g' "$JUPYTERSERVERLIBPATH/serverapp.py" +fi + +if [[ -f "$NOTEBOOKLIBPATH/notebookapp.py" ]]; then + perl -pi -e "s|(.*)\(web.Application\):|\$1\(web.Application\):\n$JUPYTER_PATCH|g" "$NOTEBOOKLIBPATH/notebookapp.py" + perl -pi -e 's|(.*)__init__\(handlers(.*)|$1__init__\(self.fix_handlers\(handlers, base_url\)$2|g' "$NOTEBOOKLIBPATH/notebookapp.py" +fi + +# Dump all env variables into file so they exist still though SSH +env | grep _ >> /etc/environment + +# Add conda bin to path +export PATH=$PATH:/opt/conda/bin +cp /root/.bashrc ~/.bash_profile +conda init + +# Need to fix directory permissions for publickey authentication +chmod 700 /projects +mkdir -p /projects/.ssh/ +chmod 700 /projects/.ssh/ +service ssh restart + +# TBD maap-py install + +VERSION=$(jupyter lab --version) +if [[ $VERSION > '2' ]] && [[ $VERSION < '3' ]]; then + SHELL=/bin/bash jupyter lab --ip=0.0.0.0 --port=3100 --allow-root --NotebookApp.token='' --NotebookApp.base_url=$PREVIEW_URL --no-browser --debug +elif [[ $VERSION > '3' ]] && [[ $VERSION < '4' ]]; then + SHELL=/bin/bash jupyter lab --ip=0.0.0.0 --port=3100 --allow-root --ContentsManager.allow_hidden=True --ServerApp.token='' --ServerApp.base_url=$PREVIEW_URL --no-browser --debug --ServerApp.disable_check_xsrf=True --collaborative +else + echo "Error! Jupyterlab version not supported." +fi diff --git a/jupyterlab3/environments.json b/jupyterlab3/environments.json new file mode 100644 index 0000000..5ba6e4d --- /dev/null +++ b/jupyterlab3/environments.json @@ -0,0 +1,43 @@ +[ + { + "environment": "dit", + "ade_server": "ade.dit.maap-project.org", + "api_server": "api.dit.maap-project.org", + "auth_server": "auth.dit.maap-project.org", + "mas_server": "repo.dit.maap-project.org", + "edsc_server": "ade.dit.maap-project.org:30052", + "workspace_bucket": "maap-dit-workspace", + "kibana_url": "https://35.88.169.231/metrics/goto/4f1115df1b6563782292fb1f75676f95", + "default_host": true + }, + { + "environment": "uat", + "ade_server": "ade.uat.maap-project.org", + "api_server": "api.uat.maap-project.org", + "auth_server": "auth.uat.maap-project.org", + "mas_server": "repo.uat.maap-project.org", + "edsc_server": "ade.uat.maap-project.org:30052", + "workspace_bucket": "maap-uat-workspace", + "default_host": false + }, + { + "environment": "pilot-ops", + "ade_server": "ade.ops.maap-project.org", + "api_server": "api.ops.maap-project.org", + "auth_server": "auth.ops.maap-project.org", + "mas_server": "mas.ops.maap-project.org", + "edsc_server": "ade.ops.maap-project.org:30052", + "workspace_bucket": "maap-ops-workspace", + "default_host": false + }, + { + "environment": "ops", + "ade_server": "ade.maap-project.org", + "api_server": "api.maap-project.org", + "auth_server": "auth.maap-project.org", + "mas_server": "mas.maap-project.org", + "edsc_server": "ade.maap-project.org:30052", + "workspace_bucket": "maap-ops-dataset", + "default_host": false + } +] diff --git a/jupyterlab3/isce3/environment.yml b/jupyterlab3/isce3/environment.yml new file mode 100644 index 0000000..ba3be7e --- /dev/null +++ b/jupyterlab3/isce3/environment.yml @@ -0,0 +1,16 @@ +name: base +channels: + - conda-forge + - nodefaults +dependencies: + - gitpython=3.1.26 + - ipyleaflet=0.17.2 + - jupyterlab=3.6.3 + - jupyterlab-git=0.34.2 + - jupyter-packaging=0.12.3 + - jupyterlab_widgets=1.0.2 + - nodejs=16.19.0 + - plotly=5.5.0 + - xmltodict=0.13.0 + - pip: + - jupyter-ydoc==0.2.4 diff --git a/jupyterlab3/jupyter_patch b/jupyterlab3/jupyter_patch new file mode 100644 index 0000000..079a2cb --- /dev/null +++ b/jupyterlab3/jupyter_patch @@ -0,0 +1,11 @@ + # Fix for Tornado's inability to handle proxy requests + from tornado.routing import _RuleList + def fix_handlers(self, handlers: _RuleList, base_url: str): + for i in range(len(handlers)): + l = list(handlers[i]) + l[0] = l[0].replace(base_url.rstrip('/'), '') + handlers[i] = tuple(l) + return handlers + + def add_handlers(self, host_pattern: str, host_handlers: _RuleList) -> None: + super().add_handlers(host_pattern, self.fix_handlers(host_handlers, self.settings['base_url'])) diff --git a/jupyterlab3/pangeo/environment.yml b/jupyterlab3/pangeo/environment.yml new file mode 100644 index 0000000..c9d0588 --- /dev/null +++ b/jupyterlab3/pangeo/environment.yml @@ -0,0 +1,15 @@ +name: base +channels: + - conda-forge + - nodefaults +dependencies: + - pangeo-notebook=2023.04.15 + - gitpython=3.1.30 + - ipyleaflet=0.17.2 + - jupyterlab=3.6.3 + - jupyterlab-git=0.34.2 + - jupyter-packaging=0.12.3 + - jupyterlab_widgets=3.0.7 + - nodejs=18.15.0 + - plotly=5.14.1 + - xmltodict=0.13.0 \ No newline at end of file diff --git a/jupyterlab3/shared/environment.yml b/jupyterlab3/shared/environment.yml new file mode 100644 index 0000000..a9d5dba --- /dev/null +++ b/jupyterlab3/shared/environment.yml @@ -0,0 +1,14 @@ +name: base +channels: + - conda-forge + - nodefaults +dependencies: + - gitpython=3.1.26 + - ipyleaflet=0.17.2 + - jupyterlab=3.6.1 + - jupyterlab-git=0.34.2 + - jupyter-packaging=0.12.3 + - jupyterlab_widgets=1.0.2 + - nodejs=16.14.2 + - plotly=5.5.0 + - xmltodict=0.13.0 \ No newline at end of file