[v1.x] Add aarch64 support (#20252)

* add aarch64 build support

* add publish dockerfile

* add options to build mkldnn w/ apl and acl

add options to build mkldnn w/ arm performance libraries and arm compute library

* remove libquadmath due to licensing issues #19053

Co-authored-by: Zhaoqi Zhu <[email protected]>

ci/docker/Dockerfile.publish.ubuntu1804_aarch64_cpu

@@ -0,0 +1,36 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
+
+FROM arm64v8/ubuntu:18.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_aarch64_publish.sh /work/
+RUN /work/ubuntu_aarch64_publish.sh
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib

ci/docker/install/ubuntu_aarch64_publish.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build on Ubuntu 18.04 LTS for LINUX CPU/GPU
+set -ex
+
+apt-get update
+apt-get install -y software-properties-common
+add-apt-repository ppa:ubuntu-toolchain-r/test -y
+add-apt-repository ppa:openjdk-r/ppa -y # Java lib
+apt-get update
+apt-get install -y git \
+    build-essential \
+    ninja-build \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    ccache \
+    unzip \
+    libtool \
+    curl \
+    wget \
+    sudo \
+    gnupg \
+    gnupg2 \
+    gnupg-agent \
+    pandoc \
+    python3 \
+    python3-pip \
+    automake \
+    pkg-config \
+    openjdk-8-jdk \
+    patchelf
+
+# gcc-10 required for -moutline-atomics flag
+apt-add-repository "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu bionic main"
+apt-get update
+apt install -y gcc-10 g++-10 gfortran-10
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 --slave /usr/bin/gfortran gfortran /usr/bin/gfortran-10
+
+curl -o apache-maven-3.3.9-bin.tar.gz -L http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
+    || curl -o apache-maven-3.3.9-bin.tar.gz -L https://search.maven.org/remotecontent?filepath=org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz
+
+tar xzf apache-maven-3.3.9-bin.tar.gz
+mkdir /usr/local/maven
+mv apache-maven-3.3.9/ /usr/local/maven/
+update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+update-ca-certificates -f
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+python3 -m pip install --upgrade pip
+
+python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 nose-timer 'numpy<2.0.0,>1.16.0' 'requests<3,>=2.20.0' scipy boto3
+
+# CMake 3.13.2+ is required
+wget https://github.com/Kitware/CMake/releases/download/v3.16.5/cmake-3.16.5.tar.gz
+tar -zxvf cmake-3.16.5.tar.gz
+cd cmake-3.16.5
+./bootstrap
+make -j$(nproc)
+sudo make install
+

config/distribution/linux_aarch64_cpu.cmake

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE "Distribution" CACHE STRING "Build type")
+set(CFLAGS "-march=armv8-a" CACHE STRING "CFLAGS")
+set(CXXFLAGS "-march=armv8-a" CACHE STRING "CXXFLAGS")
+
+set(USE_CUDA OFF CACHE BOOL "Build with CUDA support")
+set(USE_OPENCV ON CACHE BOOL "Build with OpenCV support")
+set(USE_OPENMP ON CACHE BOOL "Build with Openmp support")
+set(USE_MKL_IF_AVAILABLE OFF CACHE BOOL "Use Intel MKL if found")
+set(USE_LAPACK ON CACHE BOOL "Build with lapack support")
+set(USE_TVM_OP OFF CACHE BOOL "Enable use of TVM operator build system.")
+set(USE_SSE OFF CACHE BOOL "Build with x86 SSE instruction support")
+set(USE_F16C OFF CACHE BOOL "Build with x86 F16C instruction support")
+set(USE_DIST_KVSTORE OFF CACHE BOOL "Build with DIST_KVSTORE support")
+
+set(USE_MKLDNN ON CACHE BOOL "Build with MKL-DNN support")
+# Uncomment the following line to build MKLDNN with APL support
+# APL can be downloaded from https://developer.arm.com/tools-and-software/server-and-hpc/downloads/arm-performance-libraries
+# APL needs to be added to LD_LIBRARY_PATH
+## set(DNNL_BLAS_VENDOR “ARMPL” CACHE STRING “Build MKLDNN with Arm Performance Libraries as the BLAS library”)
+# Uncomment the following lines to build MKLDNN with ACL support
+# C++ 14 is requried to build with ACL and MKLDNN recommends building ACL from source rather than
+# using the pre-built binaries from https://github.com/ARM-software/ComputeLibrary/releases
+# If pre-built binaries are used anyways, make sure to copy and rename the appropriate binaries
+# folder from <acl_root>/lib/<binaries_folder> to <acl_root>/build
+# The resulting acl root folder should look something like:
+# LICENSE README.md arm_compute build examples include lib scripts support utils
+## set(CMAKE_CXX_STANDARD 14)
+## set(ENV{ACL_ROOT_DIR} ~/arm_compute-v21.02-bin-linux)
+## set(DNNL_AARCH64_USE_ACL ON CACHE BOOL “Build MKLDNN with Arm Compute Library integration”)

tools/dependencies/make_shared_dependencies.sh

@@ -55,13 +55,17 @@ fi
 source $DIR/libz.sh
 source $DIR/libturbojpeg.sh
 source $DIR/libpng.sh
-source $DIR/libtiff.sh
+if [[ ! $ARCH == 'aarch64' ]]; then
+    source $DIR/libtiff.sh
+fi
 source $DIR/openssl.sh
 source $DIR/curl.sh
 source $DIR/eigen.sh
 source $DIR/opencv.sh
-source $DIR/protobuf.sh
-source $DIR/cityhash.sh
+if [[ ! $ARCH == 'aarch64' ]]; then
+    source $DIR/protobuf.sh
+    source $DIR/cityhash.sh
+fi
 source $DIR/zmq.sh
 source $DIR/lz4.sh
 

tools/dependencies/openssl.sh

@@ -31,7 +31,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]];
     pushd .
     cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION
     if [[ $PLATFORM == 'linux' ]]; then
-        TARGET=linux-x86_64
+        TARGET=linux-$ARCH
     elif [[ $PLATFORM == 'darwin' ]]; then
         TARGET=darwin64-x86_64-cc
     fi

tools/pip/setup.py

@@ -28,7 +28,7 @@
 if platform.system() == 'Linux':
     sys.argv.append('--python-tag')
     sys.argv.append('py3')
-    sys.argv.append('--plat-name=manylinux2014_x86_64')
+    sys.argv.append('--plat-name=manylinux2014_'+platform.machine())
 elif platform.system() == 'Darwin':
     sys.argv.append('--python-tag')
     sys.argv.append('py3')
@@ -73,6 +73,23 @@
                 os.path.join(CURRENT_DIR, 'dmlc_tracker'))
 shutil.copy(LIB_PATH[0], os.path.join(CURRENT_DIR, 'mxnet'))
 
+package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))],
+                'dmlc_tracker': []}
+if platform.system() == 'Linux':
+    libdir, mxdir = os.path.dirname(LIB_PATH[0]), os.path.join(CURRENT_DIR, 'mxnet')
+    if os.path.exists(os.path.join(libdir, 'libgfortran.so.3')):
+        shutil.copy(os.path.join(libdir, 'libgfortran.so.3'), mxdir)
+        package_data['mxnet'].append('mxnet/libgfortran.so.3')
+    elif os.path.exists(os.path.join(libdir, 'libgfortran.so.4')):
+        shutil.copy(os.path.join(libdir, 'libgfortran.so.4'), mxdir)
+        package_data['mxnet'].append('mxnet/libgfortran.so.4')
+    else:
+        shutil.copy(os.path.join(libdir, 'libgfortran.so.5'), mxdir)
+        package_data['mxnet'].append('mxnet/libgfortran.so.5')
+    if os.path.exists(os.path.join(libdir, 'libopenblas.so.0')):
+        shutil.copy(os.path.join(libdir, 'libopenblas.so.0'), mxdir)
+        package_data['mxnet'].append('mxnet/libopenblas.so.0')
+
 # copy license and notice
 shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/licenses'),
             os.path.join(CURRENT_DIR, 'mxnet/licenses'))
@@ -148,22 +165,9 @@ def skip_markdown_comments(md):
 
 short_description += ' This version uses {0}.'.format(' and '.join(libraries))
 
-package_data = {'mxnet': [os.path.join('mxnet', os.path.basename(LIB_PATH[0]))],
-                'dmlc_tracker': []}
 if Features().is_enabled("MKLDNN"):
     shutil.copytree(os.path.join(CURRENT_DIR, 'mxnet-build/3rdparty/mkldnn/include'),
                     os.path.join(CURRENT_DIR, 'mxnet/include/mkldnn'))
-if platform.system() == 'Linux':
-    libdir, mxdir = os.path.dirname(LIB_PATH[0]), os.path.join(CURRENT_DIR, 'mxnet')
-    if os.path.exists(os.path.join(libdir, 'libgfortran.so.3')):
-        shutil.copy(os.path.join(libdir, 'libgfortran.so.3'), mxdir)
-        package_data['mxnet'].append('mxnet/libgfortran.so.3')
-    else:
-        shutil.copy(os.path.join(libdir, 'libgfortran.so.4'), mxdir)
-        package_data['mxnet'].append('mxnet/libgfortran.so.4')
-    if os.path.exists(os.path.join(libdir, 'libopenblas.so.0')):
-        shutil.copy(os.path.join(libdir, 'libopenblas.so.0'), mxdir)
-        package_data['mxnet'].append('mxnet/libopenblas.so.0')
 
 # Copy licenses and notice
 for f in os.listdir('mxnet/licenses'):

tools/staticbuild/build.sh

@@ -25,6 +25,7 @@ export CURDIR=$PWD
 export DEPS_PATH=$PWD/staticdeps
 export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]')
 export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]')
+export ARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
 
 if [[ $VARIANT == darwin* ]]; then
     export VARIANT="darwin"
@@ -48,10 +49,16 @@ else
 fi
 export MAKE="make $ADD_MAKE_FLAG"
 
-export CC="gcc -fPIC -mno-avx"
-export CXX="g++ -fPIC -mno-avx"
+if [[ $ARCH == 'aarch64' ]]; then
+    export CC="gcc -fPIC -moutline-atomics"
+    export CXX="g++ -fPIC -moutline-atomics"
+    export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/aarch64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
+else
+    export CC="gcc -fPIC -mno-avx"
+    export CXX="g++ -fPIC -mno-avx"
+    export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
+fi
 export FC="gfortran"
-export PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
 export CPATH=$DEPS_PATH/include:$CPATH
 
 if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then

tools/staticbuild/build_lib_cmake.sh

@@ -20,7 +20,12 @@
 set -eo pipefail
 
 # This script builds the libraries of mxnet.
-cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake
+if [[ $ARCH == 'aarch64' ]]; then
+    cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${ARCH}_${VARIANT}.cmake
+else
+    cmake_config=${CURDIR}/config/distribution/${PLATFORM}_${VARIANT}.cmake
+fi
+
 if [[ ! -f $cmake_config ]]; then
     >&2 echo "Couldn't find cmake config $make_config for the current settings."
     exit 1
@@ -39,14 +44,7 @@ rm -rf lib; mkdir lib;
 if [[ $PLATFORM == 'linux' ]]; then
     cp -L build/libmxnet.so lib/libmxnet.so
     cp -L staticdeps/lib/libopenblas.so lib/libopenblas.so.0
-    if [[ -f /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so ]]; then
-        cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3
-    elif [[ -f /usr/lib/x86_64-linux-gnu/libgfortran.so.3 ]]; then
-        cp -L /usr/lib/x86_64-linux-gnu/libgfortran.so.3 lib/libgfortran.so.3
-    else
-        cp -L /usr/lib/x86_64-linux-gnu/libgfortran.so.4 lib/libgfortran.so.4
-    fi
-    cp -L /usr/lib/x86_64-linux-gnu/libquadmath.so.0 lib/libquadmath.so.0
+    cp -L $(ldd lib/libmxnet.so | grep libgfortran | awk '{print $3}') lib/
 elif [[ $PLATFORM == 'darwin' ]]; then
     cp -L build/libmxnet.dylib lib/libmxnet.dylib
 fi