New images for CRAN emulations (#65)

Adds Debian and Fedora images which will be used for CRAN emulations. Signed-off-by: Dinakar <[email protected]>
insightsengineering · Nov 17, 2022 · 235d3ff · 235d3ff
1 parent cb98350
commit 235d3ff
Show file tree

Hide file tree

Showing 10 changed files with 299 additions and 81 deletions.
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -9,14 +9,28 @@ on:
       - scheduled
   workflow_dispatch:
     inputs:
+      origin:
+        description: DockerHub org or username where the base image is located
+        required: true
+        type: choice
+        default: rocker
+        options:
+          - rocker
+          - rhub
       distribution:
-        description: Rocker Distro Name. Eg. rstudio
+        description: Rocker/RHub Distro Name. Eg. rstudio or fedora-gcc-devel
         required: true
         type: choice
         default: rstudio
         options:
           - rstudio
           - rstudio-local
+          - debian-clang-devel
+          - debian-gcc-devel
+          - fedora-clang-devel
+          - fedora-gcc-devel
+          - debian-gcc-patched
+          - debian-gcc-release
       r_version:
         description: R Version
         required: true
@@ -30,6 +44,7 @@ on:
           - "4.2.0"
           - "4.2.1"
           - "4.2.2"
+          - "latest"
       bioc_version:
         description: BioConductor Release
         required: true
@@ -40,6 +55,7 @@ on:
           - "3.14"
           - "3.15"
           - "3.16"
+          - "devel"
       tag:
         description: |
           Custom Image Tag/Version. Defaults to current date in the `YYYY.MM.DD` format if unspecified.
@@ -67,18 +83,24 @@ jobs:
             fi
             echo ${var}
           }
+          ORIGIN=$(normalize ${{ github.event.inputs.origin }} ${{ github.event.client_payload.origin }})
+          ORIGIN_DISTRIBUTION=${ORIGIN//"-local"}
           DISTRIBUTION=$(normalize ${{ github.event.inputs.distribution }} ${{ github.event.client_payload.distribution }})
           R_VERSION=$(normalize ${{ github.event.inputs.r_version }} ${{ github.event.client_payload.r_version }})
           BIOC_VERSION=$(normalize ${{ github.event.inputs.bioc_version }} ${{ github.event.client_payload.bioc_version }})
           TAG=$(normalize ${{ github.event.inputs.tag }} ${{ github.event.client_payload.tag }})
           TAG_LATEST=$(normalize ${{ github.event.inputs.tag_latest }} ${{ github.event.client_payload.tag_latest }})
+          echo "::set-output name=ORIGIN::$ORIGIN"
+          echo "::set-output name=ORIGIN_DISTRIBUTION::$ORIGIN_DISTRIBUTION"
           echo "::set-output name=DISTRIBUTION::$DISTRIBUTION"
           echo "::set-output name=R_VERSION::$R_VERSION"
           echo "::set-output name=BIOC_VERSION::$BIOC_VERSION"
           echo "::set-output name=TAG::$TAG"
           echo "::set-output name=TAG_LATEST::$TAG_LATEST"
         shell: bash
     outputs:
+      origin: ${{ steps.normalizer.outputs.ORIGIN }}
+      origin_distribution: ${{ steps.normalizer.outputs.ORIGIN_DISTRIBUTION }}
       distribution: ${{ steps.normalizer.outputs.DISTRIBUTION }}
       r_version: ${{ steps.normalizer.outputs.R_VERSION }}
       bioc_version: ${{ steps.normalizer.outputs.BIOC_VERSION }}
@@ -132,6 +154,11 @@ jobs:
         run: |
           # Set Image name
           image_name="${{ needs.normalize-inputs.outputs.distribution }}_${{ needs.normalize-inputs.outputs.r_version }}_bioc_${{ needs.normalize-inputs.outputs.bioc_version }}"
+          if [[ "${{ needs.normalize-inputs.outputs.distribution }}" =~ ^debian.*|^fedora.* ]]
+          then {
+            image_name="${{ needs.normalize-inputs.outputs.distribution }}"
+          }
+          fi
 
           # Set default tag as 'YYYY.MM.DD' date if it isn't set
           tag="${{ needs.normalize-inputs.outputs.tag }}"
@@ -171,6 +198,8 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
           build-args: |
+            ORIGIN=${{ needs.normalize-inputs.outputs.origin }}
+            ORIGIN_DISTRIBUTION=${{ needs.normalize-inputs.outputs.origin_distribution }}
             DISTRIBUTION=${{ needs.normalize-inputs.outputs.distribution }}
             R_VERSION=${{ needs.normalize-inputs.outputs.r_version }}
             BIOC_VERSION=${{ needs.normalize-inputs.outputs.bioc_version }}

diff --git a/.github/workflows/scheduled.yaml b/.github/workflows/scheduled.yaml
@@ -12,12 +12,38 @@ jobs:
     strategy:
       matrix:
         image:
-          - r: '4.2.2'
+          - distro_tag: '4.2.2'
             bioc: '3.16'
             distro: rstudio-local
-          - r: '4.2.2'
+            origin: rocker
+          - distro_tag: '4.2.2'
             bioc: '3.16'
             distro: rstudio
+            origin: rocker
+          - distro_tag: 'latest'
+            bioc: 'devel'
+            distro: debian-clang-devel
+            origin: rhub
+          - distro_tag: 'latest'
+            bioc: 'devel'
+            distro: debian-gcc-devel
+            origin: rhub
+          - distro_tag: 'latest'
+            bioc: 'devel'
+            distro: fedora-clang-devel
+            origin: rhub
+          - distro_tag: 'latest'
+            bioc: 'devel'
+            distro: fedora-gcc-devel
+            origin: rhub
+          - distro_tag: 'latest'
+            bioc: '3.15'
+            distro: debian-gcc-patched
+            origin: rhub
+          - distro_tag: 'latest'
+            bioc: '3.16'
+            distro: debian-gcc-release
+            origin: rhub
 
     # Trigger steps
     steps:
@@ -33,8 +59,9 @@ jobs:
           event-type: scheduled
           client-payload: >
             {
+              "origin": "${{ matrix.image.origin }}",
               "distribution": "${{ matrix.image.distro }}",
-              "r_version": "${{ matrix.image.r }}",
+              "r_version": "${{ matrix.image.distro_tag }}",
               "bioc_version": "${{ matrix.image.bioc }}",
               "tag": "",
               "tag_latest": "true"

diff --git a/Dockerfile b/Dockerfile
@@ -1,14 +1,14 @@
 # Build arguments
 ARG ORIGIN=rocker
 ARG ORIGIN_DISTRIBUTION=rstudio
-ARG R_VERSION=4.2.0
+ARG R_VERSION=4.2.2
 
 # Fetch base image
 FROM ${ORIGIN}/${ORIGIN_DISTRIBUTION}:${R_VERSION}
 
 # Reset args in build context
 ARG DISTRIBUTION=rstudio-local
-ARG BIOC_VERSION=3.15
+ARG BIOC_VERSION=3.16
 
 # Set image metadata
 LABEL org.opencontainers.image.licenses="GPL-2.0-or-later" \

diff --git a/scripts/install_bioc.R b/scripts/install_bioc.R
@@ -6,8 +6,9 @@ args <- commandArgs(trailing = TRUE)
 bioc_version <- args[1]
 
 # Install Bioc Installer
-if (!requireNamespace("BiocManager", quietly = TRUE))
+if (!requireNamespace("BiocManager", quietly = TRUE)) {
   install.packages("BiocManager")
+}
 BiocManager::install(
   version = bioc_version,
   Ncpus = parallel::detectCores(),

diff --git a/scripts/install_bioc_pkgs.R b/scripts/install_bioc_pkgs.R
@@ -28,7 +28,13 @@ shared_pkgs <- c(
 # Per distro BioC packages to install
 bioc_pkgs <- list(
   rstudio = shared_pkgs,
-  `rstudio-local` = shared_pkgs
+  `rstudio-local` = shared_pkgs,
+  `debian-clang-devel` = shared_pkgs,
+  `debian-gcc-devel` = shared_pkgs,
+  `fedora-clang-devel` = shared_pkgs,
+  `fedora-gcc-devel` = shared_pkgs,
+  `debian-gcc-patched` = shared_pkgs,
+  `debian-gcc-release` = shared_pkgs
 )
 
 # Get diff of installed and uninstalled packages for
@@ -38,8 +44,10 @@ new_pkgs <- bioc_pkgs[[distribution]][
 ]
 
 # Install only uninstalled packages
-if (length(new_pkgs))
+if (length(new_pkgs)) {
   BiocManager::install(new_pkgs,
-                       Ncpus = parallel::detectCores(),
-                       upgrade = "never",
-                       force = TRUE)
+    Ncpus = parallel::detectCores(),
+    upgrade = "never",
+    force = TRUE
+  )
+}
diff --git a/scripts/install_cran_pkgs.R b/scripts/install_cran_pkgs.R
@@ -175,12 +175,20 @@ cran_pkgs <- list(
     shared_pkgs,
     "diffviewer",
     "languageserver"
-  )
+  ),
+  `debian-clang-devel` = shared_pkgs,
+  `debian-gcc-devel` = shared_pkgs,
+  `fedora-clang-devel` = shared_pkgs,
+  `fedora-gcc-devel` = shared_pkgs,
+  `debian-gcc-patched` = shared_pkgs,
+  `debian-gcc-release` = shared_pkgs
 )
 
 # Re-install packages with newer versions
-install.packages(reinstall_with_newer_version, type = "source",
-                 Ncpus = parallel::detectCores())
+install.packages(reinstall_with_newer_version,
+  type = "source",
+  Ncpus = parallel::detectCores()
+)
 
 # Get diff of installed and uninstalled packages for
 # idempotent package installation
@@ -189,9 +197,12 @@ new_pkgs_from_src <- cran_pkgs_from_src[[distribution]][
 ]
 
 # Install "source only" packages from source
-if (length(new_pkgs_from_src))
-  install.packages(new_pkgs_from_src, type = "source",
-                   Ncpus = parallel::detectCores())
+if (length(new_pkgs_from_src)) {
+  install.packages(new_pkgs_from_src,
+    type = "source",
+    Ncpus = parallel::detectCores()
+  )
+}
 
 
 # Get diff of installed and uninstalled packages for
@@ -201,9 +212,11 @@ new_pkgs <- cran_pkgs[[distribution]][
 ]
 
 # Install all other packages, only if they are uninstalled on the image
-if (length(new_pkgs))
+if (length(new_pkgs)) {
   install.packages(new_pkgs,
-                   Ncpus = parallel::detectCores())
+    Ncpus = parallel::detectCores()
+  )
+}
 
 # Conditionally install phantonJS
 if (require("shinytest")) {

diff --git a/scripts/install_gh_pkgs.R b/scripts/install_gh_pkgs.R
@@ -6,17 +6,22 @@ args <- commandArgs(trailing = TRUE)
 distribution <- args[1]
 
 # Packages to install
+# Regular CRAN packages to install
+shared_pkgs <- c(
+  "tlverse/[email protected]",
+  "insightsengineering/nesttemplate",
+  "openpharma/staged.dependencies@*release"
+)
+
 gh_pkgs <- list(
-  rstudio = c(
-    "tlverse/[email protected]",
-    "insightsengineering/nesttemplate",
-    "openpharma/staged.dependencies@*release"
-  ),
-  `rstudio-local` = c(
-    "tlverse/[email protected]",
-    "insightsengineering/nesttemplate",
-    "openpharma/staged.dependencies@*release"
-  )
+  rstudio = shared_pkgs,
+  `rstudio-local` = shared_pkgs,
+  `debian-clang-devel` = shared_pkgs,
+  `debian-gcc-devel` = shared_pkgs,
+  `fedora-clang-devel` = shared_pkgs,
+  `fedora-gcc-devel` = shared_pkgs,
+  `debian-gcc-patched` = shared_pkgs,
+  `debian-gcc-release` = shared_pkgs
 )
 
 # Get diff of installed and uninstalled packages for

diff --git a/scripts/install_other_pkgs.R b/scripts/install_other_pkgs.R
@@ -40,12 +40,18 @@ stat_pkgs <- c(
 
 # List for packages to be installed in a given distribution
 other_pkgs <- list(
-    rstudio = c(
-      stat_pkgs
-    ),
-    `rstudio-local` = c(
-      nest_packages
-    )
+  rstudio = c(
+    stat_pkgs
+  ),
+  `rstudio-local` = c(
+    nest_packages
+  ),
+  `debian-clang-devel` = c(),
+  `debian-gcc-devel` = c(),
+  `fedora-clang-devel` = c(),
+  `fedora-gcc-devel` = c(),
+  `debian-gcc-patched` = c(),
+  `debian-gcc-release` = c()
 )
 
 # Get diff of installed and uninstalled packages for

diff --git a/scripts/install_pip_pkgs.py b/scripts/install_pip_pkgs.py
@@ -30,6 +30,12 @@ def install(packages=[]):
     "rstudio-local": shared_packages + [
         "pre-commit",
     ],
+    "debian-clang-devel": [],
+    "debian-gcc-devel": [],
+    "fedora-clang-devel": [],
+    "fedora-gcc-devel": [],
+    "debian-gcc-patched": [],
+    "debian-gcc-release": [],
 }
 
 # Install packages