[FEATURE]Migrating all CD pipelines to Ninja build + fix cu112 CD pipeline

3rdparty/mshadow/CMakeLists.txt

@@ -42,7 +42,7 @@ else()
   target_compile_definitions(mshadow INTERFACE MSHADOW_USE_SSE=0)
 endif()
 if(USE_CUDNN)
-  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN)
+  target_compile_definitions(mshadow INTERFACE MSHADOW_USE_CUDNN=1)
 endif()
 if(MSHADOW_IN_CXX11)
   target_compile_definitions(mshadow INTERFACE MSHADOW_IN_CXX11)

CMakeLists.txt

@@ -148,6 +148,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND UNIX AND NOT APPLE)
   # Enforce DT_PATH instead of DT_RUNPATH
   set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--disable-new-dtags")
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+  set(Protobuf_USE_STATIC_LIBS ON)
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/upstream;${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")

ci/docker/Dockerfile.build.ubuntu_build_cuda

@@ -16,12 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build MXNet on Ubuntu 16.04 for GPU but on
+# Dockerfile to build MXNet on Ubuntu 18.04 for GPU but on
 # a CPU-only instance. This restriction is caused by the CPP-
 # package generation, requiring the actual CUDA library to be
 # present
 
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_c

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_jekyll

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_julia

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_lite

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_python

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_r

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_cpu_scala

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
 
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_gpu_cu100

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.0-devel-ubuntu16.04
+FROM nvidia/cuda:10.0-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_gpu_cu101

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.1-devel-ubuntu16.04
+FROM nvidia/cuda:10.1-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_gpu_cu102

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:10.2-devel-ubuntu16.04
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_gpu_cu110

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/Dockerfile.build.ubuntu_gpu_cu112

@@ -16,9 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
 
-FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu16.04
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04
 
 WORKDIR /work/deps
 

ci/docker/install/requirements

@@ -32,3 +32,4 @@ astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
 setuptools
+coverage

ci/docker/install/ubuntu_caffe.sh

@@ -40,6 +40,7 @@ git clone http://github.com/BVLC/caffe.git
 
 cd caffe
 cp Makefile.config.example Makefile.config
+echo "OPENCV_VERSION := 3" >> Makefile.config
 
 echo "CPU_ONLY := 1" >> Makefile.config
 

ci/docker/install/ubuntu_clang.sh

@@ -25,11 +25,9 @@ set -ex
 apt-get update || true
 # Install clang 3.9 (the same version as in XCode 8.*) and 6.0 (latest major release)
 wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
-    apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" && \
+    apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-6.0 main" && \
     apt-get update && \
-    apt-get install -y clang-3.9 clang-6.0 clang-tidy-6.0 && \
-    clang-3.9 --version && \
+    apt-get install -y clang-6.0 clang-tidy-6.0 && \
     clang-6.0 --version
 
 # Use llvm's master version of run-clang-tidy.py.  This version has mostly minor updates, but

ci/docker/install/ubuntu_core.sh

@@ -38,10 +38,14 @@ apt-get install -y \
     libcurl4-openssl-dev \
     libjemalloc-dev \
     libhdf5-dev \
+    libomp5 \
+    libomp-dev \
     liblapack-dev \
     libopenblas-dev \
     libopencv-dev \
-    libturbojpeg \
+    libjpeg-turbo8-dev \
+    libjpeg8-dev \
+    libturbojpeg0-dev \
     libzmq3-dev \
     libtinfo-dev \
     zlib1g-dev \
@@ -52,12 +56,12 @@ apt-get install -y \
     sudo \
     unzip \
     vim-nox \
-    default-jdk \
+    openjdk-8-jdk \
+    openjdk-8-jre \
     wget
 
 # Use libturbojpeg package as it is correctly compiled with -fPIC flag
 # https://github.com/HaxeFoundation/hashlink/issues/147
-ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so
 
 
 # CMake 3.13.2+ is required

ci/docker/install/ubuntu_llvm.sh

@@ -17,9 +17,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
+echo deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-5.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -qO - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -

ci/docker/install/ubuntu_onnx.sh

@@ -30,4 +30,4 @@ echo "Installing libprotobuf-dev and protobuf-compiler ..."
 apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
-pip3 install pytest pytest-cov pytest-xdist protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 'numpy>1.16.0,<1.19.0' gluonnlp gluoncv
+pip3 install pytest==6.2.2 pytest-cov==2.11.1 pytest-xdist==2.2.1 protobuf==3.5.2 onnx==1.7.0 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.6.0 'numpy>1.16.0,<1.19.0' gluonnlp==0.10.0 gluoncv==0.8.0

ci/docker/install/ubuntu_r.sh

@@ -27,7 +27,7 @@
 set -ex
 cd "$(dirname "$0")"
 # install libraries for mxnet's r package on ubuntu
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
+echo "deb http://cran.rstudio.com/bin/linux/ubuntu bionic-cran40/" >> /etc/apt/sources.list
 
 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
 

ci/docker/runtime_functions.sh

@@ -489,6 +489,8 @@ build_ubuntu_cpu_openblas() {
         USE_DIST_KVSTORE=1            \
         USE_LIBJPEG_TURBO=1           \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+        USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
     make cython PYTHON=python3
 }
@@ -507,6 +509,7 @@ build_ubuntu_cpu_mkl() {
         USE_INTEL_PATH=/opt/intel     \
         USE_DIST_KVSTORE=1            \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -681,6 +684,7 @@ build_ubuntu_cpu_mkldnn() {
         USE_TVM_OP=1                  \
         USE_BLAS=openblas             \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -696,6 +700,7 @@ build_ubuntu_cpu_mkldnn_mkl() {
         USE_BLAS=mkl                  \
         USE_SIGNAL_HANDLER=1          \
         USE_INTEL_PATH=/opt/intel/    \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -744,6 +749,7 @@ build_ubuntu_gpu_mkldnn() {
         USE_TVM_OP=0                              \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -761,6 +767,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
         USE_TVM_OP=0                              \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -778,6 +785,7 @@ build_ubuntu_gpu_cuda101_cudnn7() {
         USE_DIST_KVSTORE=1                        \
         CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
         USE_SIGNAL_HANDLER=1                      \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
     make cython PYTHON=python3
 }
@@ -1649,6 +1657,8 @@ build_ubuntu_cpu_docs() {
         USE_DIST_KVSTORE=1            \
         USE_LIBJPEG_TURBO=1           \
         USE_SIGNAL_HANDLER=1          \
+        USE_LAPACK_PATH=/usr/lib/x86_64-linux-gnu \
+        USE_LIBJPEG_TURBO_PATH=/usr/lib/x86_64-linux-gnu \
         -j$(nproc)
 }
 
@@ -1974,6 +1984,7 @@ build_static_libmxnet() {
     set -ex
     pushd .
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    CMAKE_STATICBUILD=1
     source tools/staticbuild/build.sh ${mxnet_variant}
     popd
 }

ci/jenkins/Jenkinsfile_clang

@@ -34,10 +34,8 @@ utils.assign_node_labels(utility: 'utility', linux_cpu: 'mxnetlinux-cpu', linux_
 utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
-    custom_steps.compile_unix_clang_3_9_cpu(),
     custom_steps.compile_unix_clang_6_cpu(),
     custom_steps.compile_unix_clang_tidy_cpu(),
-    custom_steps.compile_unix_clang_3_9_mkldnn_cpu(),
     custom_steps.compile_unix_clang_6_mkldnn_cpu()
   ]) 
 }

ci/jenkins/Jenkinsfile_miscellaneous

@@ -38,11 +38,7 @@ core_logic: {
     custom_steps.compile_unix_asan_cpu(),
     custom_steps.compile_unix_amalgamation_min(),
     custom_steps.compile_unix_amalgamation()
-  ]) 
-
-  utils.parallel_stage('Tests', [
-    custom_steps.misc_asan_cpu()
-  ]) 
+  ])
 }
 ,
 failure_handler: {

ci/jenkins/Jenkinsfile_unix_cpu

@@ -45,7 +45,6 @@ core_logic: {
 
   utils.parallel_stage('Tests', [
     custom_steps.test_unix_python3_cpu(),
-    custom_steps.test_unix_python3_debug_cpu(),
     custom_steps.test_unix_python3_mkl_cpu(),
     custom_steps.test_unix_python3_mkldnn_cpu(),
     custom_steps.test_unix_python3_mkldnn_mkl_cpu(),