diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 578466a..080ddf8 100755 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -9,14 +9,28 @@ on: - scheduled workflow_dispatch: inputs: + origin: + description: DockerHub org or username where the base image is located + required: true + type: choice + default: rocker + options: + - rocker + - rhub distribution: - description: Rocker Distro Name. Eg. rstudio + description: Rocker/RHub Distro Name. Eg. rstudio or fedora-gcc-devel required: true type: choice default: rstudio options: - rstudio - rstudio-local + - debian-clang-devel + - debian-gcc-devel + - fedora-clang-devel + - fedora-gcc-devel + - debian-gcc-patched + - debian-gcc-release r_version: description: R Version required: true @@ -30,6 +44,7 @@ on: - "4.2.0" - "4.2.1" - "4.2.2" + - "latest" bioc_version: description: BioConductor Release required: true @@ -40,6 +55,7 @@ on: - "3.14" - "3.15" - "3.16" + - "devel" tag: description: | Custom Image Tag/Version. Defaults to current date in the `YYYY.MM.DD` format if unspecified. @@ -67,11 +83,15 @@ jobs: fi echo ${var} } + ORIGIN=$(normalize ${{ github.event.inputs.origin }} ${{ github.event.client_payload.origin }}) + ORIGIN_DISTRIBUTION=${ORIGIN//"-local"} DISTRIBUTION=$(normalize ${{ github.event.inputs.distribution }} ${{ github.event.client_payload.distribution }}) R_VERSION=$(normalize ${{ github.event.inputs.r_version }} ${{ github.event.client_payload.r_version }}) BIOC_VERSION=$(normalize ${{ github.event.inputs.bioc_version }} ${{ github.event.client_payload.bioc_version }}) TAG=$(normalize ${{ github.event.inputs.tag }} ${{ github.event.client_payload.tag }}) TAG_LATEST=$(normalize ${{ github.event.inputs.tag_latest }} ${{ github.event.client_payload.tag_latest }}) + echo "::set-output name=ORIGIN::$ORIGIN" + echo "::set-output name=ORIGIN_DISTRIBUTION::$ORIGIN_DISTRIBUTION" echo "::set-output name=DISTRIBUTION::$DISTRIBUTION" echo "::set-output name=R_VERSION::$R_VERSION" echo "::set-output name=BIOC_VERSION::$BIOC_VERSION" @@ -79,6 +99,8 @@ jobs: echo "::set-output name=TAG_LATEST::$TAG_LATEST" shell: bash outputs: + origin: ${{ steps.normalizer.outputs.ORIGIN }} + origin_distribution: ${{ steps.normalizer.outputs.ORIGIN_DISTRIBUTION }} distribution: ${{ steps.normalizer.outputs.DISTRIBUTION }} r_version: ${{ steps.normalizer.outputs.R_VERSION }} bioc_version: ${{ steps.normalizer.outputs.BIOC_VERSION }} @@ -132,6 +154,11 @@ jobs: run: | # Set Image name image_name="${{ needs.normalize-inputs.outputs.distribution }}_${{ needs.normalize-inputs.outputs.r_version }}_bioc_${{ needs.normalize-inputs.outputs.bioc_version }}" + if [[ "${{ needs.normalize-inputs.outputs.distribution }}" =~ ^debian.*|^fedora.* ]] + then { + image_name="${{ needs.normalize-inputs.outputs.distribution }}" + } + fi # Set default tag as 'YYYY.MM.DD' date if it isn't set tag="${{ needs.normalize-inputs.outputs.tag }}" @@ -171,6 +198,8 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new build-args: | + ORIGIN=${{ needs.normalize-inputs.outputs.origin }} + ORIGIN_DISTRIBUTION=${{ needs.normalize-inputs.outputs.origin_distribution }} DISTRIBUTION=${{ needs.normalize-inputs.outputs.distribution }} R_VERSION=${{ needs.normalize-inputs.outputs.r_version }} BIOC_VERSION=${{ needs.normalize-inputs.outputs.bioc_version }} diff --git a/.github/workflows/scheduled.yaml b/.github/workflows/scheduled.yaml index a163007..dc640f7 100644 --- a/.github/workflows/scheduled.yaml +++ b/.github/workflows/scheduled.yaml @@ -12,12 +12,38 @@ jobs: strategy: matrix: image: - - r: '4.2.2' + - distro_tag: '4.2.2' bioc: '3.16' distro: rstudio-local - - r: '4.2.2' + origin: rocker + - distro_tag: '4.2.2' bioc: '3.16' distro: rstudio + origin: rocker + - distro_tag: 'latest' + bioc: 'devel' + distro: debian-clang-devel + origin: rhub + - distro_tag: 'latest' + bioc: 'devel' + distro: debian-gcc-devel + origin: rhub + - distro_tag: 'latest' + bioc: 'devel' + distro: fedora-clang-devel + origin: rhub + - distro_tag: 'latest' + bioc: 'devel' + distro: fedora-gcc-devel + origin: rhub + - distro_tag: 'latest' + bioc: '3.15' + distro: debian-gcc-patched + origin: rhub + - distro_tag: 'latest' + bioc: '3.16' + distro: debian-gcc-release + origin: rhub # Trigger steps steps: @@ -33,8 +59,9 @@ jobs: event-type: scheduled client-payload: > { + "origin": "${{ matrix.image.origin }}", "distribution": "${{ matrix.image.distro }}", - "r_version": "${{ matrix.image.r }}", + "r_version": "${{ matrix.image.distro_tag }}", "bioc_version": "${{ matrix.image.bioc }}", "tag": "", "tag_latest": "true" diff --git a/Dockerfile b/Dockerfile index 911fb33..9127941 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,14 @@ # Build arguments ARG ORIGIN=rocker ARG ORIGIN_DISTRIBUTION=rstudio -ARG R_VERSION=4.2.0 +ARG R_VERSION=4.2.2 # Fetch base image FROM ${ORIGIN}/${ORIGIN_DISTRIBUTION}:${R_VERSION} # Reset args in build context ARG DISTRIBUTION=rstudio-local -ARG BIOC_VERSION=3.15 +ARG BIOC_VERSION=3.16 # Set image metadata LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \ diff --git a/scripts/install_bioc.R b/scripts/install_bioc.R index c076fec..2205359 100755 --- a/scripts/install_bioc.R +++ b/scripts/install_bioc.R @@ -6,8 +6,9 @@ args <- commandArgs(trailing = TRUE) bioc_version <- args[1] # Install Bioc Installer -if (!requireNamespace("BiocManager", quietly = TRUE)) +if (!requireNamespace("BiocManager", quietly = TRUE)) { install.packages("BiocManager") +} BiocManager::install( version = bioc_version, Ncpus = parallel::detectCores(), diff --git a/scripts/install_bioc_pkgs.R b/scripts/install_bioc_pkgs.R index ff89c8e..6492eb6 100755 --- a/scripts/install_bioc_pkgs.R +++ b/scripts/install_bioc_pkgs.R @@ -28,7 +28,13 @@ shared_pkgs <- c( # Per distro BioC packages to install bioc_pkgs <- list( rstudio = shared_pkgs, - `rstudio-local` = shared_pkgs + `rstudio-local` = shared_pkgs, + `debian-clang-devel` = shared_pkgs, + `debian-gcc-devel` = shared_pkgs, + `fedora-clang-devel` = shared_pkgs, + `fedora-gcc-devel` = shared_pkgs, + `debian-gcc-patched` = shared_pkgs, + `debian-gcc-release` = shared_pkgs ) # Get diff of installed and uninstalled packages for @@ -38,8 +44,10 @@ new_pkgs <- bioc_pkgs[[distribution]][ ] # Install only uninstalled packages -if (length(new_pkgs)) +if (length(new_pkgs)) { BiocManager::install(new_pkgs, - Ncpus = parallel::detectCores(), - upgrade = "never", - force = TRUE) + Ncpus = parallel::detectCores(), + upgrade = "never", + force = TRUE + ) +} diff --git a/scripts/install_cran_pkgs.R b/scripts/install_cran_pkgs.R index a6dc645..fb905a6 100755 --- a/scripts/install_cran_pkgs.R +++ b/scripts/install_cran_pkgs.R @@ -175,12 +175,20 @@ cran_pkgs <- list( shared_pkgs, "diffviewer", "languageserver" - ) + ), + `debian-clang-devel` = shared_pkgs, + `debian-gcc-devel` = shared_pkgs, + `fedora-clang-devel` = shared_pkgs, + `fedora-gcc-devel` = shared_pkgs, + `debian-gcc-patched` = shared_pkgs, + `debian-gcc-release` = shared_pkgs ) # Re-install packages with newer versions -install.packages(reinstall_with_newer_version, type = "source", - Ncpus = parallel::detectCores()) +install.packages(reinstall_with_newer_version, + type = "source", + Ncpus = parallel::detectCores() +) # Get diff of installed and uninstalled packages for # idempotent package installation @@ -189,9 +197,12 @@ new_pkgs_from_src <- cran_pkgs_from_src[[distribution]][ ] # Install "source only" packages from source -if (length(new_pkgs_from_src)) - install.packages(new_pkgs_from_src, type = "source", - Ncpus = parallel::detectCores()) +if (length(new_pkgs_from_src)) { + install.packages(new_pkgs_from_src, + type = "source", + Ncpus = parallel::detectCores() + ) +} # Get diff of installed and uninstalled packages for @@ -201,9 +212,11 @@ new_pkgs <- cran_pkgs[[distribution]][ ] # Install all other packages, only if they are uninstalled on the image -if (length(new_pkgs)) +if (length(new_pkgs)) { install.packages(new_pkgs, - Ncpus = parallel::detectCores()) + Ncpus = parallel::detectCores() + ) +} # Conditionally install phantonJS if (require("shinytest")) { diff --git a/scripts/install_gh_pkgs.R b/scripts/install_gh_pkgs.R index 1ad36bf..e9e808c 100755 --- a/scripts/install_gh_pkgs.R +++ b/scripts/install_gh_pkgs.R @@ -6,17 +6,22 @@ args <- commandArgs(trailing = TRUE) distribution <- args[1] # Packages to install +# Regular CRAN packages to install +shared_pkgs <- c( + "tlverse/sl3@v1.4.4", + "insightsengineering/nesttemplate", + "openpharma/staged.dependencies@*release" +) + gh_pkgs <- list( - rstudio = c( - "tlverse/sl3@v1.4.4", - "insightsengineering/nesttemplate", - "openpharma/staged.dependencies@*release" - ), - `rstudio-local` = c( - "tlverse/sl3@v1.4.4", - "insightsengineering/nesttemplate", - "openpharma/staged.dependencies@*release" - ) + rstudio = shared_pkgs, + `rstudio-local` = shared_pkgs, + `debian-clang-devel` = shared_pkgs, + `debian-gcc-devel` = shared_pkgs, + `fedora-clang-devel` = shared_pkgs, + `fedora-gcc-devel` = shared_pkgs, + `debian-gcc-patched` = shared_pkgs, + `debian-gcc-release` = shared_pkgs ) # Get diff of installed and uninstalled packages for diff --git a/scripts/install_other_pkgs.R b/scripts/install_other_pkgs.R index 8759401..e47cb64 100755 --- a/scripts/install_other_pkgs.R +++ b/scripts/install_other_pkgs.R @@ -40,12 +40,18 @@ stat_pkgs <- c( # List for packages to be installed in a given distribution other_pkgs <- list( - rstudio = c( - stat_pkgs - ), - `rstudio-local` = c( - nest_packages - ) + rstudio = c( + stat_pkgs + ), + `rstudio-local` = c( + nest_packages + ), + `debian-clang-devel` = c(), + `debian-gcc-devel` = c(), + `fedora-clang-devel` = c(), + `fedora-gcc-devel` = c(), + `debian-gcc-patched` = c(), + `debian-gcc-release` = c() ) # Get diff of installed and uninstalled packages for diff --git a/scripts/install_pip_pkgs.py b/scripts/install_pip_pkgs.py index 8d18c95..6e39e30 100755 --- a/scripts/install_pip_pkgs.py +++ b/scripts/install_pip_pkgs.py @@ -30,6 +30,12 @@ def install(packages=[]): "rstudio-local": shared_packages + [ "pre-commit", ], + "debian-clang-devel": [], + "debian-gcc-devel": [], + "fedora-clang-devel": [], + "fedora-gcc-devel": [], + "debian-gcc-patched": [], + "debian-gcc-release": [], } # Install packages diff --git a/scripts/install_sysdeps.sh b/scripts/install_sysdeps.sh index 970e86b..c72c51c 100755 --- a/scripts/install_sysdeps.sh +++ b/scripts/install_sysdeps.sh @@ -7,10 +7,11 @@ set -e distribution="$1" # Hash map of system deps -declare -A pkgs_to_install +declare -A pkgs_to_install_debian +declare -A pkgs_to_install_fedora -# Shared deps -shared_deps="\ +# Shared deps for debian +shared_deps_debian="\ libxml2-dev \ pandoc \ libicu-dev \ @@ -30,7 +31,7 @@ curl \ qpdf \ unattended-upgrades \ ssh \ -libmysqlclient-dev \ +libmariadb-dev \ libsodium-dev \ default-jdk \ cmake \ @@ -38,55 +39,177 @@ graphviz \ libaio1 \ alien \ libxss1 \ +git \ +libssl-dev \ " -# Deps specific to rstudio -pkgs_to_install["rstudio"]="${shared_deps} \ +# Shared deps for fedora +shared_deps_fedora="\ +libxml2-devel \ +pandoc \ +libicu-devel \ +libgit2-devel \ +zlib-devel \ +libfonts \ +freetype-devel \ +libjpeg-turbo-devel \ +libpng-devel \ +libtiff-devel \ +fribidi-devel \ +harfbuzz-devel \ +ImageMagick \ +ImageMagick-c++-devel\ +unixODBC-devel \ +libcurl-devel \ +qpdf \ +dnf-automatic \ +libssh-devel \ +openssh \ +mariadb-devel \ +libsodium-devel \ +java-11-openjdk \ +cmake \ +graphviz \ +libxslt-devel \ +chromium-headless \ +openssl-devel \ +git-all \ +" + +# Deps specific on the rstudio image +pkgs_to_install_debian["rstudio"]="${shared_deps_debian} \ jags \ " -# Deps specific to rstudio-local -pkgs_to_install["rstudio-local"]="${shared_deps} \ +# Deps specific on the rstudio-local image +pkgs_to_install_debian["rstudio-local"]="${shared_deps_debian} \ vim \ xdg-utils \ python3-pip \ less \ " -# Set env vars -export DEBIAN_FRONTEND=noninteractive -export ACCEPT_EULA=Y - -# Update -apt-get update -y - -# Install packages -# expected word splitting - list of packages require it -# shellcheck disable=SC2086 -apt-get install -q -y ${pkgs_to_install["${distribution}"]} - -# Add Chrome repo and install Chrome -wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - -echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list -apt-get update -y -apt-get install -q -y google-chrome-stable -rm /etc/apt/sources.list.d/google.list - -# Install quarto -ARCH=$(dpkg --print-architecture) -QUARTO_DL_URL=$(wget -qO- https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest | grep -oP "(?<=\"browser_download_url\":\s\")https.*${ARCH}\.deb") -wget -q "${QUARTO_DL_URL}" -O quarto-"${ARCH}".deb -dpkg -i quarto-"${ARCH}".deb -quarto check install - -# Install security patches -unattended-upgrade -v - -# Clean up -apt-get autoremove -y -apt-get autoclean -y -rm -rf /var/lib/apt/lists/* quarto-"${ARCH}".deb - -# Purge and recreate locales -locale-gen --purge en_US.UTF-8 -echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale +# Deps specific on the debian-clang-devel image +pkgs_to_install_debian["debian-clang-devel"]="${shared_deps_debian} \ +jags \ +" + +# Deps specific on the debian-gcc-devel image +pkgs_to_install_debian["debian-gcc-devel"]="${shared_deps_debian} \ +jags \ +" + +# Deps specific on the debian-gcc-patched image +pkgs_to_install_debian["debian-gcc-patched"]="${shared_deps_debian} \ +jags \ +" + +# Deps specific on the debian-gcc-release image +pkgs_to_install_debian["debian-gcc-release"]="${shared_deps_debian} \ +jags \ +" + +# Deps specific on the fedora-gcc-devel image +pkgs_to_install_fedora["fedora-gcc-devel"]="${shared_deps_fedora}" + +# Deps specific on the fedora-clang-devel image +pkgs_to_install_fedora["fedora-clang-devel"]="${shared_deps_fedora}" + +# Perform installations for debian distros +if [[ "$distribution" =~ ^rstudio.*|^debian.* ]] +then { + # Set env vars + export DEBIAN_FRONTEND=noninteractive + export ACCEPT_EULA=Y + + # Update + apt-get update -y + + # Install packages + # expected word splitting - list of packages require it + # shellcheck disable=SC2086 + apt-get install -q -y ${pkgs_to_install_debian["${distribution}"]} + + # Add Chrome repo and install Chrome + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list + apt-get update -y + apt-get install -q -y google-chrome-stable + rm /etc/apt/sources.list.d/google.list + + # Install quarto + ARCH=$(dpkg --print-architecture) + QUARTO_DL_URL=$(wget -qO- https://api.github.com/repos/quarto-dev/quarto-cli/releases/latest | grep -oP "(?<=\"browser_download_url\":\s\")https.*${ARCH}\.deb") + wget -q "${QUARTO_DL_URL}" -O quarto-"${ARCH}".deb + dpkg -i quarto-"${ARCH}".deb + quarto check install + + # Install security patches + unattended-upgrade -v + + # Clean up + apt-get autoremove -y + apt-get autoclean -y + rm -rf /var/lib/apt/lists/* quarto-"${ARCH}".deb + + # Purge and recreate locales] + if [[ "$distribution" =~ ^rstudio.* ]] + then { + locale-gen --purge en_US.UTF-8 + echo -e 'LANG="en_US.UTF-8"\nLANGUAGE="en_US:en"\n' > /etc/default/locale + + # Also override the default repository used in the rstudio images + echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" > \ + /usr/local/lib/R/etc/Rprofile.site + } + fi +} +fi + +if [[ "$distribution" =~ ^fedora.* ]] +then { + # Update + dnf update -y + + # Ugrade + dnf upgrade --refresh -y + + # Install packages + # expected word splitting - list of packages require it + # shellcheck disable=SC2086 + dnf install -q -y ${pkgs_to_install_fedora["${distribution}"]} + + # Clean up + dnf autoremove -y + dnf clean all +} +fi + +# Symlink R if it's in a non-default path +if [ -d "/opt/R-devel/bin/" ] +then { + ln -s /opt/R-devel/bin/R /usr/bin/R + ln -s /opt/R-devel/bin/Rscript /usr/bin/Rscript + # Also set default CRAN repo + echo "options(repos=structure(c(CRAN='https://cloud.r-project.org')))" > \ + /opt/R-devel/lib64/R/etc/Rprofile.site +} +fi + +if [ -d "/opt/R-patched/bin/" ] +then { + ln -s /opt/R-patched/bin/R /usr/bin/R + ln -s /opt/R-patched/bin/Rscript /usr/bin/Rscript + # Also set default CRAN repo + echo "options(repos=structure(c(CRAN='https://cloud.r-project.org')))" > \ + /opt/R-patched/lib64/R/etc/Rprofile.site +} +fi + +# Set default initializer if unavailable +if [ ! -f /init ] +then { + echo "sh" > /init + chmod +x /init +} +fi