[FEATURE]Migrating all CD pipelines to Ninja build + fix cu112 CD pip…

…eline (apache#19974) * migrating cd builds to ninja + removing static links to nvidia libs and leagacy cuda versions * installing NCCL manually for cuda11.2 container * set MSHADOW_USE_CUDNN=1 in CMakelists of mshadow to build properly for CUDNN support * adding coverage to cd requirements file to fix cu100, cu101 and cu102 tests * updating cd_test containers to ubuntu 18 * adding cmake config for linux native and adding USE_KV_STORE in linux_cpu * updating zmq builds to statically link to libmxnet.so * updating toolchains for r, clang and llvm for ubuntu18. OpenBlas Static link for 'distribution' build type only. Fix caffe build to use openCV 3. Remove leagacy Clang 3.9 from CI * fix versions for pip install in ubuntu_core_sh add new search path for cuDNN * finxing cudnn link problem for CUDA<=11.0 * adding library paths for libjpegturbo and lapack to fix failing CI on ubuntu 18 images * removing ASAN integration test from miscellaneous CI as its not required * fix lapack path for gpu builds * correctly installing libjpegturbo for ubuntu 18 * updating docker images of r,jekyll,julia etc test containers+ fix java version to 8 * installing libomp.so * removing debug test as its not required. Code clean-up * adding alternate URL source for MNIST dataset as original website is down * skipping flaky tests issue tracked apache#20011 Co-authored-by: Rohit Kumar Srivastava <[email protected]>
access2rohit · Mar 12, 2021 · 1765b5c · 1765b5c
1 parent 408471a
commit 1765b5c
Show file tree

Hide file tree

Showing 41 changed files with 195 additions and 218 deletions.
diff --git a/3rdparty/mshadow/CMakeLists.txt b/3rdparty/mshadow/CMakeLists.txt
@@ -42,7 +42,7 @@ else()
   target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE=0)
 endif()
 if(USE_CUDNN)
-  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN)
+  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN=1)
 endif()
 if(MSHADOW_IN_CXX11)
   target_compile_definitions(mshadow INTERFACE MSHADOW_IN_CXX11)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -142,6 +142,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND UNIX AND NOT APPLE)
   # Enforce DT_PATH instead of DT_RUNPATH
   set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags")
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+  set(Protobuf_USE_STATIC_LIBS ON)
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")

diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -16,12 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build MXNet on Ubuntu 16.04 for GPU but on
+# Dockerfile to build MXNet on Ubuntu 18.04 for GPU but on
 # a CPU-only instance. This restriction is caused by the CPP-
 # package generation, requiring the actual CUDA library to be
 # present
 
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_c b/ci/docker/Dockerfile.build.ubuntu_cpu_c
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll b/ci/docker/Dockerfile.build.ubuntu_cpu_jekyll
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_julia b/ci/docker/Dockerfile.build.ubuntu_cpu_julia
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_lite b/ci/docker/Dockerfile.build.ubuntu_cpu_lite
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_python b/ci/docker/Dockerfile.build.ubuntu_cpu_python
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_r b/ci/docker/Dockerfile.build.ubuntu_cpu_r
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu_scala b/ci/docker/Dockerfile.build.ubuntu_cpu_scala
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu100 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.0-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu101 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu101
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.2-devel-ubuntu16.04
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu110 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu110
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu112 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu112
@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:11.2-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements
@@ -31,4 +31,5 @@ pylint==2.3.1  # pylint and astroid need to be aligned
 astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
-setuptools<50
+setuptools
+coverage
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh
@@ -40,6 +40,7 @@ git clone http://github.com/BVLC/caffe.git
 
 cd caffe
 cp Makefile.config.example Makefile.config
+echo "OPENCV_VERSION := 3" >> Makefile.config
 
 echo "CPU_ONLY := 1" >> Makefile.config
 

diff --git a/ci/docker/install/ubuntu_clang.sh b/ci/docker/install/ubuntu_clang.sh
@@ -25,11 +25,9 @@ set -ex
 apt-get update || true
 # Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major release)
 wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
-    apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" && \
+    apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main" && \
     apt-get update && \
-    apt-get install -y clang-3.9 clang-6.0 clang-tidy-6.0 && \
-    clang-3.9 --version && \
+    apt-get install -y clang-6.0 clang-tidy-6.0 && \
     clang-6.0 --version
 
 # Use llvm's master version of run-clang-tidy.py.  This version has mostly minor updates, but

diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
@@ -38,10 +38,14 @@ apt-get install -y \
     libcurl4-openssl-dev \
     libjemalloc-dev \
     libhdf5-dev \
+    libomp5 \
+    libomp-dev \
     liblapack-dev \
     libopenblas-dev \
     libopencv-dev \
-    libturbojpeg \
+    libjpeg-turbo8-dev \
+    libjpeg8-dev \
+    libturbojpeg0-dev \
     libzmq3-dev \
     libtinfo-dev \
     zlib1g-dev \
@@ -52,11 +56,12 @@ apt-get install -y \
     sudo \
     unzip \
     vim-nox \
+    openjdk-8-jdk \
+    openjdk-8-jre \
     wget
 
 # Use libturbojpeg package as it is correctly compiled with -fPIC flag
 # https://github.com/HaxeFoundation/hashlink/issues/147
-ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so
 
 
 # CMake 3.13.2+ is required

diff --git a/ci/docker/install/ubuntu_llvm.sh b/ci/docker/install/ubuntu_llvm.sh
@@ -17,9 +17,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -

diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/docker/install/ubuntu_onnx.sh
@@ -30,5 +30,4 @@ echo "Installing libprotobuf-dev and protobuf-compiler ..."
 apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
-echo "Installing pytest, pytest-cov, protobuf, Pillow, ONNX, tabulate and onnxruntime..."
-pip3 install pytest==3.6.3 pytest-cov==2.5.1 protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.4.0
+pip3 install pytest==6.2.2 pytest-cov==2.11.1 pytest-xdist==2.2.1 protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 'numpy>1.16.0,<1.19.0' gluonnlp==0.10.0 gluoncv==0.8.0
diff --git a/ci/docker/install/ubuntu_r.sh b/ci/docker/install/ubuntu_r.sh
@@ -27,7 +27,7 @@
 set -ex
 cd "$(dirname "$0")"
 # install libraries for mxnet's r package on ubuntu
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
+echo "deb http://cran.rstudio.com/bin/linux/ubuntu bionic-cran40/" >> /etc/apt/sources.list
 
 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
 

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
@@ -489,6 +489,8 @@ build_ubuntu_cpu_openblas() {
         USE_DIST_KVSTORE=1            \
         USE_LIBJPEG_TURBO=1           \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+        USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
     make cython PYTHON=python3
 }
@@ -506,6 +508,7 @@ build_ubuntu_cpu_mkl() {
         USE_INTEL_PATH=/opt/intel     \
         USE_DIST_KVSTORE=1            \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -680,6 +683,7 @@ build_ubuntu_cpu_mkldnn() {
         USE_TVM_OP=1                  \
         USE_BLAS=openblas             \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -695,6 +699,7 @@ build_ubuntu_cpu_mkldnn_mkl() {
         USE_BLAS=mkl                  \
         USE_SIGNAL_HANDLER=1          \
         USE_INTEL_PATH=/opt/intel/    \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -768,6 +773,7 @@ build_ubuntu_gpu_mkldnn() {
         USE_TVM_OP=0                              \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -785,6 +791,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
         USE_TVM_OP=0                              \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -802,6 +809,7 @@ build_ubuntu_gpu_cuda101_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
     make cython PYTHON=python3
 }
@@ -1659,6 +1667,8 @@ build_ubuntu_cpu_docs() {
         USE_DIST_KVSTORE=1            \
         USE_LIBJPEG_TURBO=1           \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+        USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -1984,6 +1994,7 @@ build_static_libmxnet() {
     set -ex
     pushd .
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    CMAKE_STATICBUILD=1
     source tools/staticbuild/build.sh ${mxnet_variant}
     popd
 }

diff --git a/ci/jenkins/Jenkinsfile_clang b/ci/jenkins/Jenkinsfile_clang
@@ -34,10 +34,8 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_clang_3_9_cpu(),
     custom_steps.compile_unix_clang_6_cpu(),
     custom_steps.compile_unix_clang_tidy_cpu(),
-    custom_steps.compile_unix_clang_3_9_mkldnn_cpu(),
     custom_steps.compile_unix_clang_6_mkldnn_cpu()
   ]) 
 }

diff --git a/ci/jenkins/Jenkinsfile_miscellaneous b/ci/jenkins/Jenkinsfile_miscellaneous
@@ -38,11 +38,7 @@ core_logic: {
     custom_steps.compile_unix_asan_cpu(),
     custom_steps.compile_unix_amalgamation_min(),
     custom_steps.compile_unix_amalgamation()
-  ]) 
-
-  utils.parallel_stage('Tests', [
-    custom_steps.misc_asan_cpu()
-  ]) 
+  ])
 }
 ,
 failure_handler: {

diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu
@@ -45,7 +45,6 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_unix_python3_cpu(),
-    custom_steps.test_unix_python3_debug_cpu(),
     custom_steps.test_unix_python3_mkl_cpu(),
     custom_steps.test_unix_python3_mkldnn_cpu(),
     custom_steps.test_unix_python3_mkldnn_mkl_cpu(),