migrate RStudio (#107)

* migrate RStudio * fix JDKs * add PG 17 * update sync jobs
QPod · Oct 30, 2024 · ba99187 · ba99187
1 parent f2d3737
commit ba99187
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 132 deletions.
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
@@ -2,17 +2,15 @@ name: build-docker-images
 
 on:
   push:
-    branches: [ main ]
-    paths-ignore:
-      - "*.md"
+    branches: [ "main" ]
+    paths-ignore: [ "*.md" ]
 
   pull_request:
-    branches: [ main ]
-    paths-ignore:
-      - "*.md"
+    branches: [ "main" ]
+    paths-ignore: [ "*.md" ]
+
+  workflow_dispatch:  # Allows you to run this workflow manually from the Actions tab
 
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
 
 env:
   BUILDKIT_PROGRESS: "plain"  # Full logs for CI build.
@@ -44,9 +42,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: |
-          source ./tool.sh          
-          build_image python-3.7  latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7"  && push_image
+          source ./tool.sh
           build_image python-3.11 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.11" && push_image
+          build_image python-3.7  latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7"  && push_image          
 
 
   qpod_core:
@@ -59,7 +57,7 @@ jobs:
           source ./tool.sh
           build_image core latest  docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \
-            --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \
+            --build-arg "ARG_PROFILE_R=base,datascience" \
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JAVA=base,maven" \
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
@@ -118,20 +116,6 @@ jobs:
       - uses: actions/checkout@v4
       - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image
 
-  qpod_r-studio:
-    name: 'r-studio'
-    needs: qpod_base
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          source ./tool.sh
-          build_image r-studio  latest docker_core/Dockerfile \
-            --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \
-            --build-arg "ARG_PROFILE_JAVA=base" \
-            --build-arg "ARG_PROFILE_LATEX=base,cjk"
-          push_image
-
   qpod_node:
     name: 'node'
     needs: qpod_base
@@ -141,18 +125,18 @@ jobs:
       - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image
 
   qpod_jdk:
-    name: 'jdk,jdk-8,jdk-11,jdk-17'
+    name: 'jdk,jdk-8,jdk-11,jdk-17,jdk-21'
     needs: qpod_base
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: |
           source ./tool.sh
+          build_image jdk-21 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"  --build-arg "VERSION_JDK=21"
           build_image jdk-17 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"  --build-arg "VERSION_JDK=17"
           build_image jdk-11 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"  --build-arg "VERSION_JDK=11"
           build_image jdk-8  latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base"  --build-arg "VERSION_JDK=8"
-          alias_image jdk-11 latest jdk latest
-          push_image jdk
+          alias_image jdk-11 latest jdk latest && push_image jdk
 
   qpod_julia:
     name: 'julia'
@@ -192,7 +176,7 @@ jobs:
           source ./tool.sh
           build_image data-science-stack latest  docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_PYTHON=base,database" \
-            --build-arg "ARG_PROFILE_R=base,rstudio,rshiny" \
+            --build-arg "ARG_PROFILE_R=base,datascience" \
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JULIA=base"
           push_image data-science-stack
@@ -207,7 +191,7 @@ jobs:
           source ./tool.sh
           build_image full-stack latest  docker_core/Dockerfile \
             --build-arg "ARG_PROFILE_PYTHON=base,database" \
-            --build-arg "ARG_PROFILE_R=base,rstudio,rshiny" \
+            --build-arg "ARG_PROFILE_R=base,datascience" \
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JAVA=base,maven" \
             --build-arg "ARG_PROFILE_JULIA=base" \
@@ -292,7 +276,7 @@ jobs:
           alias_image torch-cuda118 latest torch latest
           push_image
 
-  qpod_torch_cuda112:
+  qpod_torch_cuda123:
     name: 'torch-cuda123'
     needs: qpod_cuda_123
     runs-on: ubuntu-latest
@@ -339,17 +323,6 @@ jobs:
           alias_image py-nlp-cuda118 latest py-nlp latest
           push_image
 
-  qpod_py-nlp-cuda112:
-    name: 'py-nlp-cuda112'
-    needs: qpod_cuda_112
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          source ./tool.sh
-          build_image py-nlp-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp"
-          push_image
-
 
   qpod_py-cv:
     name: 'py-cv'
@@ -374,24 +347,26 @@ jobs:
           build_image full-cuda-11.8 latest docker_core/Dockerfile \
             --build-arg "BASE_IMG=cuda_11.8" \
             --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \
-            --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \
+            --build-arg "ARG_PROFILE_R=base,datascience" \
             --build-arg "ARG_PROFILE_NODEJS=base" \
             --build-arg "ARG_PROFILE_JAVA=base,maven" \
             --build-arg "ARG_PROFILE_LATEX=base,cjk"
           alias_image full-cuda-11.8 latest core-cuda latest && push_image cuda
 
   ## Postgres
   qpod_postgres:
-    name: 'postgres-15,postgres-16'
+    name: 'postgres-17,postgres-16,postgres-15'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: |
           source ./tool.sh
-          build_image postgres-15 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:15"
-          build_image postgres-15 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-15" --build-arg "PYTHON_VERSION=3.12"
+          build_image postgres-17 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:17"
+          build_image postgres-17 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-17" --build-arg "PYTHON_VERSION=3.12"
           build_image postgres-16 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:16"
           build_image postgres-16 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-16" --build-arg "PYTHON_VERSION=3.12"
+          build_image postgres-15 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:15"
+          build_image postgres-15 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-15" --build-arg "PYTHON_VERSION=3.12"
           push_image postgres-
 
   ## DockerKit - including image-syncer and docker-compsoe

diff --git a/docker_atom/work/script-setup.sh b/docker_atom/work/script-setup.sh
@@ -95,30 +95,31 @@ setup_nvtop() {
 
 
 setup_java_base() {
+  ## 23, 21(LTS); 17, 11, 8
+
   local VER_JDK=${VERSION_JDK:-"11"}
   ARCH="x64"
+  IS_ALPINE=$(grep -q 'ID=alpine' /etc/os-release && echo true || echo false)
+
   echo "Use env var VERSION_JDK to specify JDK major version. If not specified, will install version 11 by default."
   echo "Will install JDK version ${VER_JDK}"
 
-  JDK_PAGE_DOWNLOAD="https://www.oracle.com/java/technologies/downloads/" \
-  && JDK_URL_ORCA=$(curl -sL ${JDK_PAGE_DOWNLOAD} | grep "tar.gz" | grep "http" | grep -v sha256 | grep ${ARCH} | grep -i $(uname) | sed "s/'/\"/g" | sed -n 's/.*="\([^"]*\).*/\1/p' | grep "jdk-${VER_JDK}" | head -n 1)
+     PAGE_JDK_DOWNLOAD="https://www.oracle.com/java/technologies/downloads/" \
+  && URL_JDK_ORCA=$(curl -sL $PAGE_JDK_DOWNLOAD | grep "tar.gz" | grep "http" | grep -v sha256 | grep ${ARCH} | grep -i $(uname) | grep -oP "(https?://[^\s<>\'\"]*)" | grep "jdk-${VER_JDK}" | head -n 1) \
+  && VER_JDK_MINOR=$(echo $URL_JDK_ORCA | grep -Po '[\d\.]{3,}' | head -n1)
 
-  JDK_PAGE_RELEASE="https://www.oracle.com/java/technologies/javase/${VER_JDK}u-relnotes.html" \
-  && JDK_VER_MINOR=$(curl -sL "${JDK_PAGE_RELEASE}" | grep -P 'JDK \d..\d+' | grep -Po '[\d\.]{3,}' | head -n1) \
-  && JDK_URL_MSFT="https://aka.ms/download-jdk/microsoft-jdk-${JDK_VER_MINOR}-linux-${ARCH}.tar.gz"
-
-  if [ "$VER_JDK" -gt 11 ] ; then
-    URL_OPENJDK=${JDK_URL_ORCA}
-  elif [ "$VER_JDK" -gt 8 ] ; then
-    URL_OPENJDK=${JDK_URL_MSFT}
+  if [ "$VER_JDK" -gt 20 ] ; then
+    URL_JDK_DOWNLOAD=${URL_JDK_ORCA}
   else
-    echo "ORCA JDK8 download URL ref: ${JDK_URL_ORCA}"
-    URL_OPENJDK="https://javadl.oracle.com/webapps/download/GetFile/1.8.0_361-b09/0ae14417abb444ebb02b9815e2103550/linux-i586/jdk-8u361-linux-${ARCH}.tar.gz"
+       URL_JDK_adoptium="https://api.github.com/repos/adoptium/temurin${VER_JDK}-binaries/releases/latest" \
+    && URL_JDK_DOWNLOAD=$(
+      curl -sL $URL_JDK_adoptium | grep 'tar.gz' | grep -vE '.sha256|.sig|.json|debug|test' | grep ${ARCH} | grep -i $(uname) \
+      | grep -oP "(https?://[^\s<>\'\"]*)" | grep -E $(if [ "$IS_ALPINE" = true ]; then echo 'alpine'; else echo -v 'alpine'; fi) | head -n1
+    ) ;
   fi
 
-     echo "Installing JDK version ${VER_JDK} from: ${URL_OPENJDK}" \
-  && install_tar_gz "${URL_OPENJDK}" && mv /opt/jdk* /opt/jdk \
-  && ln -sf /opt/jdk/bin/* /usr/bin/
+  echo "Installing JDK version ${VER_JDK} from: ${URL_JDK_DOWNLOAD}" ;
+  install_tar_gz "${URL_JDK_DOWNLOAD}" && mv /opt/jdk* /opt/jdk && ln -sf /opt/jdk/bin/* /usr/bin/ ;
 
   type java  && echo "@ Version of Java (java):  $(java -version)"  || return -1 ;
   type javac && echo "@ Version of Java (javac): $(javac -version)" || return -1 ;
@@ -137,9 +138,9 @@ setup_java_maven() {
 
 setup_node() {
      ARCH="x64" \
-  && NODEJS_VERSION=$(curl -sL https://github.com/nodejs/node/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[.\d]+') \
-  && NODEJS_VERSION_MAJOR=$(echo "${NODEJS_VERSION}" | cut -d '.' -f1 ) \
-  && NODEJS_URL="https://nodejs.org/download/release/latest-v${NODEJS_VERSION_MAJOR}.x/node-v${NODEJS_VERSION}-linux-${ARCH}.tar.gz" \
+  && VER_NODEJS=$(curl -sL https://github.com/nodejs/node/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[.\d]+') \
+  && VER_NODEJS_MAJOR=$(echo "${VER_NODEJS}" | cut -d '.' -f1 ) \
+  && NODEJS_URL="https://nodejs.org/download/release/latest-v${VER_NODEJS_MAJOR}.x/node-v${VER_NODEJS}-linux-${ARCH}.tar.gz" \
   && echo "Downloading NodeJS from: ${NODEJS_URL}" \
   && install_tar_gz ${NODEJS_URL} \
   && mv /opt/node* /opt/node \
@@ -180,6 +181,19 @@ setup_rust() {
 }
 
 
+setup_R_base() {
+     curl -sL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \
+  && echo "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" > /etc/apt/sources.list.d/cran.list \
+  && install_apt  /opt/utils/install_list_R_base.apt \
+  && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \
+  && R -e "install.packages(c('devtools'),clean=T,quiet=T);" \
+  && R -e "install.packages(c('devtools'),clean=T,quiet=F);" \
+  && ( type java && type R && R CMD javareconf || true ) ;
+
+  type R && echo "@ Version of R: $(R --version)" || return -1 ;
+}
+
+
 setup_julia() {
      JULIA_URL="https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz" \
   && install_tar_gz $JULIA_URL \

diff --git a/docker_core/work/script-setup-R.sh b/docker_core/work/script-setup-R.sh
@@ -1,74 +1,6 @@
 source /opt/utils/script-utils.sh
 
 
-setup_R_base() {
-     curl -sL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc \
-  && echo "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" > /etc/apt/sources.list.d/cran.list \
-  && install_apt  /opt/utils/install_list_R_base.apt \
-  && echo "options(repos=structure(c(CRAN=\"https://cloud.r-project.org\")))" >> /etc/R/Rprofile.site \
-  && R -e "install.packages(c('devtools'),clean=T,quiet=T);" \
-  && R -e "install.packages(c('devtools'),clean=T,quiet=F);" \
-  && ( type java && type R && R CMD javareconf || true ) ;
-
-  type R && echo "@ Version of R: $(R --version)" || return -1 ;
-}
-
-
-setup_R_rstudio() {
-  #  https://posit.co/download/rstudio-server/
-     RSTUDIO_VERSION=$(curl -sL https://download2.rstudio.org/current.ver | cut -d'.' -f'1-3' | sed 's/+/-/g' ) \
-  && RSTUDIO_URL="https://download2.rstudio.org/server/jammy/amd64/rstudio-server-${RSTUDIO_VERSION}-amd64.deb" \
-  && curl -sL -o /tmp/rstudio.deb ${RSTUDIO_URL} \
-  && dpkg -x /tmp/rstudio.deb /tmp && mv /tmp/usr/lib/rstudio-server/ /opt/ \
-  && ln -sf /opt/rstudio-server         /usr/lib/ \
-  && ln -sf /opt/rstudio-server/bin/rs* /usr/bin/
-
-  # Allow RStudio server run as root user
-  # Configuration to make RStudio server disable authentication and do not run as daemon
-     mkdir -pv /etc/rstudio \
-  && echo "server-daemonize=0"     >> /etc/rstudio/rserver.conf \
-  && echo "server-user=root"       >> /etc/rstudio/rserver.conf \
-  && echo "auth-none=1"            >> /etc/rstudio/rserver.conf \
-  && echo "auth-minimum-user-id=0" >> /etc/rstudio/rserver.conf \
-  && echo "auth-validate-users=0"  >> /etc/rstudio/rserver.conf \
-  && echo "www-allow-origin=*"     >> /etc/rstudio/rserver.conf \
-  && echo "www-same-site=none"     >> /etc/rstudio/rserver.conf \
-  && echo "www-frame-origin=same"  >> /etc/rstudio/rserver.conf \
-  && echo "www-verify-user-agent=0">> /etc/rstudio/rserver.conf \
-  && echo "database-config-file=/etc/rstudio/db.conf"  >> /etc/rstudio/rserver.conf \
-  && echo "provider=sqlite"                            >> /etc/rstudio/db.conf \
-  && echo "directory=/etc/rstudio/"                    >> /etc/rstudio/db.conf \
-  && printf "USER=root rserver --www-port=8888" > /usr/local/bin/start-rstudio.sh \
-  && chmod u+x /usr/local/bin/start-rstudio.sh ;
-
-  type rstudio-server && echo "@ Version of rstudio-server: $(rstudio-server version)" || return -1 ;
-}
-
-
-setup_R_rshiny() {
-  #  https://posit.co/download/shiny-server/
-     RSHINY_VERSION=$(curl -sL https://s3.amazonaws.com/rstudio-shiny-server-os-build/ubuntu-18.04/x86_64/VERSION) \
-  && RSHINY_URL="https://download3.rstudio.org/ubuntu-18.04/x86_64/shiny-server-${RSHINY_VERSION}-amd64.deb" \
-  && curl -sL -o /tmp/rshiny.deb ${RSHINY_URL} \
-  && dpkg -i /tmp/rshiny.deb \
-  && sed  -i "s/run_as shiny;/run_as root;/g"  /etc/shiny-server/shiny-server.conf \
-  && sed  -i "s/3838/8888/g"                   /etc/shiny-server/shiny-server.conf \
-  && printf "USER=root shiny-server" > /usr/local/bin/start-shiny-server.sh \
-  && chmod u+x /usr/local/bin/start-shiny-server.sh
-
-  # Remove shiny's pandoc and pandoc-proc to reduce size if they are already installed in the jpy-latex step.
-     ( which pandoc          && rm -rf /opt/shiny-server/ext/pandoc/pandoc          || true ) \
-  && ( which pandoc-citeproc && rm -rf /opt/shiny-server/ext/pandoc/pandoc-citeproc || true ) \
-  && rm -rf /opt/shiny-server/ext/node/bin/shiny-server \
-  && ln -sf /opt/shiny-server/ext/node/bin/node /opt/shiny-server/ext/node/bin/shiny-server
-
-  # hack shiny-server to allow run in root user: https://github.com/rstudio/shiny-server/pull/391
-  sed  -i "s/throw new Error/logger.warn/g"  /opt/shiny-server/lib/worker/app-worker.js
-
-  type shiny-server && echo "@ Version of shiny-server: $(shiny-server --version)" || return -1 ;
-}
-
-
 setup_R_datascience() {
      install_apt  /opt/utils/install_list_R_datascience.apt \
   && install_R    /opt/utils/install_list_R_datascience.R