From 102afb1b3cd09f95497c4337ba90960a4a66b0e7 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 3 Aug 2023 13:55:47 -0700 Subject: [PATCH] [NodeJS18 Upgrade] Add opensearch rockylinux ci docker images changes (#3836) Signed-off-by: Peter Zhu --- ...opensearch-dashboards.x64.arm64.dockerfile | 103 ++++++++++++++++++ ...ockylinux8.opensearch.x64.arm64.dockerfile | 4 +- ...opensearch-dashboards.x64.arm64.dockerfile | 0 ...ld.centos7.opensearch.x64.arm64.dockerfile | 2 +- ...tos7.performance-test.x64.arm64.dockerfile | 0 5 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile rename docker/ci/dockerfiles/{current => legacy}/build.centos7.opensearch-dashboards.x64.arm64.dockerfile (100%) rename docker/ci/dockerfiles/{current => legacy}/build.centos7.opensearch.x64.arm64.dockerfile (99%) rename docker/ci/dockerfiles/{current => legacy}/test.centos7.performance-test.x64.arm64.dockerfile (100%) diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile new file mode 100644 index 0000000000..77ac08095c --- /dev/null +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch-dashboards.x64.arm64.dockerfile @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This is a docker image specifically for standardize the ci/cd environment +# for both developers and ci/cd tools in OpenSearch / OpenSearch-Dashboards +# Please read the README.md file for all the information before using this dockerfile + +FROM rockylinux:8 + +# Ensure localedef running correct with root permission +USER 0 + +# Add normal dependencies +RUN dnf clean all && dnf install -y 'dnf-command(config-manager)' && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ + dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ + dnf update -y && \ + dnf install -y which curl git gnupg2 tar net-tools procps-ng python39 python39-devel python39-pip zip unzip jq gh + +# Tools setup +COPY --chown=0:0 config/yq-setup.sh /tmp +RUN /tmp/yq-setup.sh + +# Create user group +RUN groupadd -g 1000 opensearch && \ + useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \ + mkdir -p /usr/share/opensearch && \ + chown -R 1000:1000 /usr/share/opensearch + +# Add Python dependencies +RUN dnf install -y @development zlib-devel bzip2 bzip2-devel readline-devel sqlite sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + +# Add Dashboards dependencies +RUN dnf install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib + +# Add Notebook dependencies +RUN dnf install -y nss xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc fontconfig freetype && yum clean all + +# Add Yarn dependencies +RUN dnf groupinstall -y "Development Tools" && dnf clean all && rm -rf /var/cache/dnf/* + +# Setup Shared Memory +RUN chmod -R 777 /dev/shm + +# Install PKG builder dependencies with rvm +RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \ + curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import - && \ + curl -sSL https://get.rvm.io | bash -s stable + +# Switch shell for rvm related commands +SHELL ["/bin/bash", "-lc"] + +# Install ruby / rpm / fpm related dependencies +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all + +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin +ENV RVM_HOME=/usr/local/rvm/bin +ENV GEM_HOME=/usr/share/opensearch/.gem +ENV GEM_PATH=$GEM_HOME +ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH + +# Install Python binary +RUN update-alternatives --set python /usr/bin/python3.9 && \ + update-alternatives --set python3 /usr/bin/python3.9 && \ + pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.22.12 + +# Preparation for awscliv2 +#RUN pip3 install git+https://github.com/aws/aws-cli.git@2.11.17 +#ENV AWS_CLI_FILE_ENCODING=UTF-8 + +# Change User +USER 1000 +WORKDIR /usr/share/opensearch + +# Install fpm for opensearch dashboards core +RUN gem install fpm -v 1.14.2 +ENV PATH=/usr/share/opensearch/.gem/gems/fpm-1.14.2/bin:$PATH + +# Hard code node version and yarn version for now +# nvm environment variables +ENV NVM_DIR /usr/share/opensearch/.nvm +ENV NODE_VERSION 10.24.1 +ARG NODE_VERSION_LIST="10.24.1 14.19.1 14.20.0 14.20.1 14.21.3 16.20.0 18.16.0" +# install nvm +# https://github.com/creationix/nvm#install-script +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash +# install node and npm +COPY --chown=1000:1000 config/yarn-version.sh /tmp +RUN source $NVM_DIR/nvm.sh && \ + for node_version in $NODE_VERSION_LIST; do nvm install $node_version; npm install -g yarn@`/tmp/yarn-version.sh main`; done +# add node and npm to path so the commands are available +ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH +# We use the version test to check if packages installed correctly +# And get added to the PATH +# This will fail the docker build if any of the packages not exist +RUN node -v +RUN npm -v +RUN yarn -v +RUN fpm -v diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile index dd4574d6b5..89af112145 100644 --- a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile @@ -79,9 +79,9 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \ # Add k-NN Library dependencies # EL8 requires install config-manager and enable powertools to consume openblas-static -RUN dnf install epel-release -y && dnf repolist && \ - dnf install -y 'dnf-command(config-manager)' && \ +RUN dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --set-enabled powertools && \ + dnf install epel-release -y && dnf repolist && \ dnf install openblas-static lapack gcc-gfortran -y && dnf clean all RUN pip3 install cmake==3.23.3 diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.centos7.opensearch-dashboards.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.centos7.opensearch-dashboards.x64.arm64.dockerfile diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile similarity index 99% rename from docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile index ec46156328..56bd2edeae 100644 --- a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/legacy/build.centos7.opensearch.x64.arm64.dockerfile @@ -89,7 +89,7 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \ # Add k-NN Library dependencies RUN yum install epel-release -y && yum repolist && yum install openblas-static lapack gcc-gfortran -y RUN pip3 install cmake==3.23.3 -# Upgrade gcc8 +# Upgrade gcc # The setup part is partially based on Austin Dewey's article: # https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/ RUN yum install -y centos-release-scl && yum install -y devtoolset-7 && yum clean all && \ diff --git a/docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile b/docker/ci/dockerfiles/legacy/test.centos7.performance-test.x64.arm64.dockerfile similarity index 100% rename from docker/ci/dockerfiles/current/test.centos7.performance-test.x64.arm64.dockerfile rename to docker/ci/dockerfiles/legacy/test.centos7.performance-test.x64.arm64.dockerfile