From d1e4806165477c423bc635de4ca40e8740f606c6 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 18:24:16 -0800 Subject: [PATCH 1/6] fix jdk in az docker --- docker/zoo/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/zoo/Dockerfile b/docker/zoo/Dockerfile index 6679419bc6e..5d061696ae6 100644 --- a/docker/zoo/Dockerfile +++ b/docker/zoo/Dockerfile @@ -29,6 +29,8 @@ ARG RUNTIME_DRIVER_MEMORY=20g ARG RUNTIME_EXECUTOR_CORES=4 ARG RUNTIME_EXECUTOR_MEMORY=20g ARG RUNTIME_TOTAL_EXECUTOR_CORES=4 +ARG JDK_VERSION=8u192 +ARG JDK_URL=your_jdk_url ENV ANALYTICS_ZOO_VERSION ${ANALYTICS_ZOO_VERSION} ENV SPARK_VERSION ${SPARK_VERSION} ENV BIGDL_VERSION ${BIGDL_VERSION} @@ -48,11 +50,12 @@ RUN apt-get update --fix-missing && \ apt-get install -y libsm6 #java -RUN wget https://enos.itcollege.ee/~jpoial/allalaadimised/jdk8/jdk-8u281-linux-x64.tar.gz && \ - gunzip jdk-8u281-linux-x64.tar.gz && \ - tar -xf jdk-8u281-linux-x64.tar -C /opt && \ - rm jdk-8u281-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_281 /opt/jdk +RUN wget $JDK_URL && \ + gunzip jdk-$JDK_VERSION-linux-x64.tar.gz && \ + tar -xf jdk-$JDK_VERSION-linux-x64.tar -C /opt && \ + rm jdk-$JDK_VERSION-linux-x64.tar && \ + mv /opt/jdk* /opt/jdk$JDK_VERSION && \ + ln -s /opt/jdk$JDK_VERSION /opt/jdk #python RUN apt-get install -y python3-minimal && \ From a8c8486a29dda72b29b38a70e77357479b2f0d57 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 21:16:18 -0800 Subject: [PATCH 2/6] fix jdk for hyperzoo --- docker/hyperzoo/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/hyperzoo/Dockerfile b/docker/hyperzoo/Dockerfile index 97dd4fbbe80..fca01ab53d2 100644 --- a/docker/hyperzoo/Dockerfile +++ b/docker/hyperzoo/Dockerfile @@ -6,6 +6,8 @@ ARG ANALYTICS_ZOO_VERSION=0.10.0-SNAPSHOT ARG BIGDL_VERSION=0.12.1 ARG SPARK_VERSION=2.4.3 ARG FLINK_VERSION=1.10.0 +ARG JDK_VERSION=8u192 +ARG JDK_URL=your_jdk_url ENV FLINK_HOME /opt/flink-${FLINK_VERSION} ENV ANALYTICS_ZOO_VERSION ${ANALYTICS_ZOO_VERSION} @@ -55,11 +57,12 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py37_4.8.3-Linux-x86_64. bash Miniconda3-py37_4.8.3-Linux-x86_64.sh -b && \ rm -f Miniconda3-py37_4.8.3-Linux-x86_64.sh && \ #java - wget https://enos.itcollege.ee/~jpoial/allalaadimised/jdk8/jdk-8u281-linux-x64.tar.gz && \ - gunzip jdk-8u281-linux-x64.tar.gz && \ - tar -xf jdk-8u281-linux-x64.tar -C /opt && \ - rm jdk-8u281-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_281 /opt/jdk && \ + wget $JDK_URL && \ + gunzip jdk-$JDK_VERSION-linux-x64.tar.gz && \ + tar -xf jdk-$JDK_VERSION-linux-x64.tar -C /opt && \ + rm jdk-$JDK_VERSION-linux-x64.tar && \ + mv /opt/jdk* /opt/jdk$JDK_VERSION && \ + ln -s /opt/jdk$JDK_VERSION /opt/jdk && \ #spark wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \ tar -zxvf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \ From 5288668017d09bcc6bd09bbeff3ee6f1436566a9 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 21:24:31 -0800 Subject: [PATCH 3/6] fix jdk in jenkins docker --- docker/jenkins-ci/jenkins-master/Dockerfile | 14 +++++++++----- docker/jenkins-ci/jenkins-slave/Dockerfile | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docker/jenkins-ci/jenkins-master/Dockerfile b/docker/jenkins-ci/jenkins-master/Dockerfile index b9fc0ed3269..936f83682ba 100755 --- a/docker/jenkins-ci/jenkins-master/Dockerfile +++ b/docker/jenkins-ci/jenkins-master/Dockerfile @@ -4,6 +4,9 @@ MAINTAINER The Analytics-Zoo Authors https://github.com/intel-analytics/analytic WORKDIR /opt/work +ARG JDK_VERSION=8u192 +ARG JDK_URL=your_jdk_url + ENV JAVA_HOME /opt/work/jdk ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false ENV JENKINS_HOME /opt/work/jenkins @@ -23,11 +26,12 @@ COPY jenkins.sh ${JENKINS_HOME}/jenkins.sh RUN apt-get update && apt-get install -y git wget curl && rm -rf /var/lib/apt/lists/* #java -RUN wget https://enos.itcollege.ee/~jpoial/allalaadimised/jdk8/jdk-8u281-linux-x64.tar.gz && \ - gunzip jdk-8u281-linux-x64.tar.gz && \ - tar -xf jdk-8u281-linux-x64.tar -C /opt && \ - rm jdk-8u281-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_281 /opt/jdk +RUN wget $JDK_URL && \ + gunzip jdk-$JDK_VERSION-linux-x64.tar.gz && \ + tar -xf jdk-$JDK_VERSION-linux-x64.tar -C /opt && \ + rm jdk-$JDK_VERSION-linux-x64.tar && \ + mv /opt/jdk* /opt/jdk$JDK_VERSION && \ + ln -s /opt/jdk$JDK_VERSION /opt/jdk # Install Jenkins Master RUN curl -fsSL ${JENKINS_URL} -o ${JENKINS_HOME}/jenkins.war && \ diff --git a/docker/jenkins-ci/jenkins-slave/Dockerfile b/docker/jenkins-ci/jenkins-slave/Dockerfile index f44d34c5c4b..af5ea95f5a2 100644 --- a/docker/jenkins-ci/jenkins-slave/Dockerfile +++ b/docker/jenkins-ci/jenkins-slave/Dockerfile @@ -2,6 +2,9 @@ FROM ubuntu:16.04 WORKDIR /opt/work +ARG JDK_VERSION=8u192 +ARG JDK_URL=your_jdk_url + ENV JAVA_8_HOME /opt/work/jdk8 ENV JAVA_HOME ${JAVA_8_HOME} ENV SCALA_HOME /opt/work/scala @@ -30,11 +33,12 @@ RUN apt-get update --fix-missing && \ apt-get install -y libgtk2.0-dev #jdk8 -RUN wget https://enos.itcollege.ee/~jpoial/allalaadimised/jdk8/jdk-8u281-linux-x64.tar.gz && \ - gunzip jdk-8u281-linux-x64.tar.gz && \ - tar -xf jdk-8u281-linux-x64.tar -C /opt && \ - rm jdk-8u281-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_281 /opt/jdk && \ +RUN wget $JDK_URL && \ + gunzip jdk-$JDK_VERSION-linux-x64.tar.gz && \ + tar -xf jdk-$JDK_VERSION-linux-x64.tar -C /opt && \ + rm jdk-$JDK_VERSION-linux-x64.tar && \ + mv /opt/jdk* /opt/jdk$JDK_VERSION && \ + ln -s /opt/jdk$JDK_VERSION /opt/jdk && \ #scala wget https://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.tgz && \ gunzip scala-2.11.8.tgz && \ From c35094b15087277c52f950c4572a3c8d70f67619 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 21:27:52 -0800 Subject: [PATCH 4/6] fix jdk in cluster serving docker --- docker/cluster-serving/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/cluster-serving/Dockerfile b/docker/cluster-serving/Dockerfile index aa53a4e1777..e7221f1fd47 100644 --- a/docker/cluster-serving/Dockerfile +++ b/docker/cluster-serving/Dockerfile @@ -8,6 +8,8 @@ ARG SPARK_VERSION=2.4.3 ARG BIGDL_VERSION=0.12.1 ARG ANALYTICS_ZOO_VERSION=0.9.0 ARG FLINK_VERSION=1.11.2 +ARG JDK_VERSION=8u192 +ARG JDK_URL=your_jdk_url ENV FLINK_HOME /opt/work/flink-${FLINK_VERSION} ENV JAVA_HOME /opt/jdk From 79800ac42a1a7b1407b3c75fbeb670d4cceae697 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 21:34:03 -0800 Subject: [PATCH 5/6] fix jdk --- docker/cluster-serving/Dockerfile | 11 ++++++----- docker/zoo/README.md | 13 +++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docker/cluster-serving/Dockerfile b/docker/cluster-serving/Dockerfile index e7221f1fd47..1135c9306ce 100644 --- a/docker/cluster-serving/Dockerfile +++ b/docker/cluster-serving/Dockerfile @@ -31,11 +31,12 @@ RUN apt-get install -y python3-minimal && \ pip install analytics-zoo-serving #java -RUN wget https://enos.itcollege.ee/~jpoial/allalaadimised/jdk8/jdk-8u281-linux-x64.tar.gz && \ - gunzip jdk-8u281-linux-x64.tar.gz && \ - tar -xf jdk-8u281-linux-x64.tar -C /opt && \ - rm jdk-8u281-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_281 /opt/jdk +RUN wget $JDK_URL && \ + gunzip jdk-$JDK_VERSION-linux-x64.tar.gz && \ + tar -xf jdk-$JDK_VERSION-linux-x64.tar -C /opt && \ + rm jdk-$JDK_VERSION-linux-x64.tar && \ + mv /opt/jdk* /opt/jdk$JDK_VERSION && \ + ln -s /opt/jdk$JDK_VERSION /opt/jdk #redis RUN wget http://download.redis.io/releases/redis-5.0.5.tar.gz && \ diff --git a/docker/zoo/README.md b/docker/zoo/README.md index b0d31b67dea..81307df16ad 100644 --- a/docker/zoo/README.md +++ b/docker/zoo/README.md @@ -35,15 +35,21 @@ ## How to build it. -### By default, you can build a Analytics-Zoo:default image with latest nightly-build Analytics-Zoo distributions: +### By default, you can build a Analytics-Zoo:default image with latest nightly-build Analytics-Zoo distributions, you also need to provide jdk version and jdk download url: - sudo docker build --rm -t intelanalytics/analytics-zoo:default . + sudo docker build \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --rm -t intelanalytics/analytics-zoo:default . ### If you need http and https proxy to build the image: sudo docker build \ --build-arg http_proxy=http://your-proxy-host:your-proxy-port \ --build-arg https_proxy=https://your-proxy-host:your-proxy-port \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --build-arg no_proxy=x.x.x.x \ --rm -t intelanalytics/analytics-zoo:default . ### You can also specify the ANALYTICS_ZOO_VERSION and SPARK_VERSION to build a specific Analytics-Zoo image: @@ -51,6 +57,9 @@ sudo docker build \ --build-arg http_proxy=http://your-proxy-host:your-proxy-port \ --build-arg https_proxy=https://your-proxy-host:your-proxy-port \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --build-arg no_proxy=x.x.x.x \ --build-arg ANALYTICS_ZOO_VERSION=0.3.0 \ --build-arg BIGDL_VERSION=0.6.0 \ --build-arg SPARK_VERSION=2.3.1 \ From b229b6891bf36b68ecba729faba7174211275837 Mon Sep 17 00:00:00 2001 From: glorysdj Date: Sun, 7 Feb 2021 21:40:02 -0800 Subject: [PATCH 6/6] fix readme --- docker/jenkins-ci/Readme.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docker/jenkins-ci/Readme.md b/docker/jenkins-ci/Readme.md index e7c816609c3..9293932695e 100755 --- a/docker/jenkins-ci/Readme.md +++ b/docker/jenkins-ci/Readme.md @@ -6,13 +6,20 @@ Jenkins has master nodes and slave agents. To make Jenkins well functioning, mas ### 1.1 Build master First, you should step into the directory jenkins-master and run ```bash -sudo docker build --rm -t intelanalytics/jenkins-master . +sudo docker build \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --rm -t intelanalytics/jenkins-master . ``` If proxy is needed for building the image, building arguments related to proxy should be added in the command above. For example, ```bash -sudo docker build --build-arg http_proxy=http://yourproxyhost:port \ - --build-arg https_proxy=https://yourproxyhost:port \ - --rm -t intelanalytics/jenkins-master . +sudo docker build \ + --build-arg http_proxy=http://yourproxyhost:port \ + --build-arg https_proxy=https://yourproxyhost:port \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --build-arg no_proxy=x.x.x.x \ + --rm -t intelanalytics/jenkins-master . ``` Note that you can set any tag of the image by changine the parameter after the option -t. @@ -25,13 +32,20 @@ There are some extra files other than Dockerfile in the directory jenkins-master ### 1.2 Build slaves Building Jenkins slave is similar to building Jenkins master. First, step into the directory jenkins-slave and run ```bash -sudo docker build --rm -t intelanalytics/jenkins-slave . +sudo docker build \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --rm -t intelanalytics/jenkins-slave . ``` If proxy is needed for building the image, the following command should be run. ```bash -sudo docker build --build-arg http_proxy=http://yourproxyhost:port \ - --build-arg https_proxy=https://yourproxyhost:port \ - --rm -t intelanalytics/jenkins-slave . +sudo docker build \ + --build-arg http_proxy=http://yourproxyhost:port \ + --build-arg https_proxy=https://yourproxyhost:port \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=http://your-http-url-to-download-jdk \ + --build-arg no_proxy=x.x.x.x \ + --rm -t intelanalytics/jenkins-slave . ``` There are some extra files other than Dockerfile in the directory jenkins-slave. + pip.conf: provide configurations of pip