Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
Update edge toolchain
Browse files Browse the repository at this point in the history
  • Loading branch information
@leezu @nickguletskii
leezu authored and nickguletskii committed May 20, 2020
1 parent 38ec873 commit bcfa364
Show file tree
Hide file tree
Showing 18 changed files with 435 additions and 705 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ endif(MSVC)

if(NOT mxnet_LINKER_LIBS)
set(mxnet_LINKER_LIBS "")
endif(NOT mxnet_LINKER_LIBS)
endif()

if(USE_GPROF)
message(STATUS "Using GPROF")
Expand Down
11 changes: 9 additions & 2 deletions ci/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
return "nvidia-docker" if use_nvidia_docker else "docker"


def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str:
def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool,
cache_intermediate: bool) -> str:
"""
Build a container for the given platform
:param platform: Platform
Expand Down Expand Up @@ -104,6 +105,8 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
"--build-arg", "GROUP_ID={}".format(os.getgid())]
if no_cache:
cmd.append("--no-cache")
if cache_intermediate:
cmd.append("--rm=false")
elif registry:
cmd.extend(["--cache-from", tag])
cmd.extend(["-t", tag, get_dockerfiles_path()])
Expand Down Expand Up @@ -330,6 +333,9 @@ def main() -> int:
parser.add_argument("--no-cache", action="store_true",
help="passes --no-cache to docker build")

parser.add_argument("--cache-intermediate", action="store_true",
help="passes --rm=false to docker build")

parser.add_argument("-e", "--environment", nargs="*", default=[],
help="Environment variables for the docker container. "
"Specify with a list containing either names or name=value")
Expand Down Expand Up @@ -361,7 +367,8 @@ def main() -> int:
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
if not args.run_only:
build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry,
num_retries=args.docker_build_retries, no_cache=args.no_cache)
num_retries=args.docker_build_retries, no_cache=args.no_cache,
cache_intermediate=args.cache_intermediate)
else:
logging.info("Skipping docker build step.")

Expand Down
94 changes: 36 additions & 58 deletions ci/docker/Dockerfile.build.android_armv7
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,68 +18,46 @@
#
# Dockerfile to build MXNet for Android ARMv7

FROM dockcross/base
MAINTAINER Pedro Larroy "[email protected]"

# The cross-compiling emulator
RUN apt-get update && apt-get install -y \
unzip

ENV CROSS_TRIPLE=arm-linux-androideabi
ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld

ENV ANDROID_NDK_REVISION 17b
ENV ANDROID_NDK_API 27
ENV ANDROID_NDK_ARCH arm
WORKDIR /work/deps
COPY install/android_ndk.sh /work/deps
RUN /work/deps/android_ndk.sh

ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm

# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
ARG VCS_REF
ARG VCS_URL
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name=$IMAGE \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url=$VCS_URL \
org.label-schema.schema-version="1.0"


ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++

WORKDIR /work/deps

COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
WORKDIR /work
COPY install/ubuntu_arm.sh /work/
RUN /work/ubuntu_arm.sh

COPY install/arm_openblas.sh /work/
COPY install/android_armv7_openblas.sh /work/deps
RUN /work/deps/android_armv7_openblas.sh

ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}

WORKDIR /work
FROM ubuntu:20.04

ENV ARCH=armv7l \
HOSTCC=gcc \
HOSTCXX=g++ \
TARGET=ARMV7

WORKDIR /usr/local

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential \
ninja-build \
cmake \
ccache \
git \
curl \
unzip \
&& rm -rf /var/lib/apt/lists/*

RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip && \
unzip android-ndk-r19-linux-x86_64.zip && \
rm android-ndk-r19-linux-x86_64.zip
ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake

RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
mkdir /usr/local/openblas-android && \
cd /usr/local/OpenBLAS && \
export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \
make NOFORTRAN=1 ARM_SOFTFP_ABI=1 \
LDFLAGS="-L/usr/local/android-ndk-r19/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/lib/gcc/arm-linux-androideabi/4.9.x -lm" \
CC=$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang AR=$TOOLCHAIN/bin/arm-linux-androideabi-ar && \
make PREFIX=/usr/local/openblas-android install && \
cd /usr/local && \
rm -rf OpenBLAS
ENV OpenBLAS_HOME=/usr/local/openblas-android

ARG USER_ID=0
ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh

COPY runtime_functions.sh /work/
WORKDIR /work/mxnet

WORKDIR /work/build
92 changes: 35 additions & 57 deletions ci/docker/Dockerfile.build.android_armv8
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,68 +18,46 @@
#
# Dockerfile to build MXNet for Android ARM64/ARMv8

FROM dockcross/base
MAINTAINER Pedro Larroy "[email protected]"

RUN apt-get update && apt-get install -y \
unzip

WORKDIR /work/deps

# Build x86 dependencies.
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh

# Setup Android cross-compilation environment.
ENV CROSS_TRIPLE=aarch64-linux-android
ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld


ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm

# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
ARG VCS_REF
ARG VCS_URL
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name=$IMAGE \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url=$VCS_URL \
org.label-schema.schema-version="1.0"

ENV ARCH aarch64
ENV ANDROID_NDK_REVISION 17b
ENV ANDROID_NDK_API 27
ENV ANDROID_NDK_ARCH arm64
WORKDIR /work/deps
COPY install/android_ndk.sh /work/deps
RUN /work/deps/android_ndk.sh


WORKDIR /work/deps
COPY install/android_ndk.sh /work/
RUN /work/android_ndk.sh

ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++

# Build ARM dependencies.
COPY install/android_arm64_openblas.sh /work/
RUN /work/android_arm64_openblas.sh
ENV CPLUS_INCLUDE_PATH /work/deps/OpenBLAS
FROM ubuntu:20.04

ENV ARCH=aarch64 \
HOSTCC=gcc \
HOSTCXX=g++ \
TARGET=ARMV8

WORKDIR /usr/local

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential \
ninja-build \
cmake \
ccache \
git \
curl \
unzip \
&& rm -rf /var/lib/apt/lists/*

RUN curl -o android-ndk-r19-linux-x86_64.zip -L https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip && \
unzip android-ndk-r19-linux-x86_64.zip && \
rm android-ndk-r19-linux-x86_64.zip
ENV CMAKE_TOOLCHAIN_FILE=/usr/local/android-ndk-r19/build/cmake/android.toolchain.cmake

RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
mkdir /usr/local/openblas-android && \
cd /usr/local/OpenBLAS && \
export TOOLCHAIN=/usr/local/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 && \
make NOFORTRAN=1 \
LDFLAGS="-L/usr/local/android-ndk-r21/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/lib/gcc/aarch64-linux-android/4.9.x -lm" \
CC=$TOOLCHAIN/bin/aarch64-linux-android21-clang AR=$TOOLCHAIN/bin/aarch64-linux-android-ar && \
make PREFIX=/usr/local/openblas-android install && \
cd /usr/local && \
rm -rf OpenBLAS
ENV OpenBLAS_HOME=/usr/local/openblas-android

ARG USER_ID=0
ARG GROUP_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh

COPY runtime_functions.sh /work/

WORKDIR /work/build
45 changes: 31 additions & 14 deletions ci/docker/Dockerfile.build.armv6
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,42 @@
#
# Dockerfile to build MXNet for ARMv6

FROM dockcross/linux-armv6
FROM ubuntu:20.04

ENV ARCH armv6l
ENV HOSTCC gcc
ENV TARGET ARMV6
ENV ARCH=armv6l \
HOSTCC=gcc \
HOSTCXX=g++ \
TARGET=ARMV6

WORKDIR /work/deps
WORKDIR /usr/local

COPY install/ubuntu_arm.sh /work/
RUN /work/ubuntu_arm.sh
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential \
ninja-build \
cmake \
ccache \
git \
curl \
zip \
python3 \
python3-pip \
&& rm -rf /var/lib/apt/lists/*

COPY install/arm_openblas.sh /work/
RUN /work/arm_openblas.sh
# We use a toolchain from toolchains.bootlin.com instead of Debian / Ubunut
# crossbuild-essential-armel toolchain, as the latter targets ARM architecture
# versions 4T, 5T, and 6, whereas we only wish to target ARMV6 and like to use
# ARMV6 specific features. https://wiki.debian.org/ArmEabiPort
RUN curl -o armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 -L https://toolchains.bootlin.com/downloads/releases/toolchains/armv6-eabihf/tarballs/armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 && \
tar xf armv6-eabihf--glibc--stable-2020.02-2.tar.bz2 && \
rm armv6-eabihf--glibc--stable-2020.02-2.tar.bz2
ENV CMAKE_TOOLCHAIN_FILE=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/share/buildroot/toolchainfile.cmake

ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}

COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
cd /usr/local/OpenBLAS && \
make NOFORTRAN=1 CC=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/bin/arm-linux-gcc && \
make PREFIX=/usr/local/armv6-eabihf--glibc--stable-2020.02-2/arm-buildroot-linux-gnueabihf/sysroot install && \
cd /usr/local && \
rm -rf OpenBLAS

ARG USER_ID=0
ARG GROUP_ID=0
Expand Down
54 changes: 33 additions & 21 deletions ci/docker/Dockerfile.build.armv7
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,39 @@
# specific language governing permissions and limitations
# under the License.
#
# Dockerfile to build MXNet for Android ARMv7

FROM dockcross/linux-armv7

ENV ARCH armv7l
ENV HOSTCC gcc
ENV TARGET ARMV7

WORKDIR /work/deps

COPY install/ubuntu_arm.sh /work/
RUN /work/ubuntu_arm.sh

COPY install/arm_openblas.sh /work/
RUN /work/arm_openblas.sh

ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}

COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
# Dockerfile to build MXNet for ARMv7

FROM ubuntu:20.04

ENV ARCH=armv7l \
HOSTCC=gcc \
HOSTCXX=g++ \
TARGET=ARMV7

WORKDIR /usr/local

RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential \
ninja-build \
cmake \
ccache \
git \
curl \
zip \
python3 \
python3-pip \
crossbuild-essential-armhf \
&& rm -rf /var/lib/apt/lists/*

COPY toolchains/arm-linux-gnueabihf-toolchain.cmake /usr/local
ENV CMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf-toolchain.cmake

RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
cd /usr/local/OpenBLAS && \
make NOFORTRAN=1 CC=arm-linux-gnueabihf-gcc && \
make PREFIX=/usr/local/arm-linux-gnueabihf install && \
cd /usr/local && \
rm -rf OpenBLAS

ARG USER_ID=0
ARG GROUP_ID=0
Expand Down
Loading

0 comments on commit bcfa364

Please sign in to comment.