Merge remote-tracking branch 'mxnet/master' into tol_improvements

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: 'Bug'
+labels: 'Bug, needs triage'
 assignees: ''
 
 ---

README.md

@@ -40,7 +40,7 @@ deep learning systems, and interesting insights of DL systems for hackers.
 
 Ask Questions
 -------------
-* Please use [discuss.mxnet.io](https://discuss.mxnet.io/) for asking questions.
+* Please use [discuss.d2l.ai](https://discuss.d2l.ai/c/d2l-en/mxnet/) or [old version:discuss.mxnet.io](https://discuss.mxnet.io/) for asking questions. 
 * Please use [mxnet/issues](https://github.com/apache/incubator-mxnet/issues) for reporting bugs.
 * [Frequent Asked Questions](https://mxnet.apache.org/faq/faq.html)
 
@@ -80,6 +80,7 @@ What's New
 
 Contents
 --------
+* [Book](https://d2l.ai)
 * [Website](https://mxnet.apache.org)
 * [Documentation](https://mxnet.apache.org/api)
 * [Blog](https://mxnet.apache.org/blog)

ci/build.py

@@ -288,9 +288,17 @@ def list_platforms() -> str:
     return "\nSupported platforms:\n{}".format('\n'.join(get_platforms()))
 
 
-def load_docker_cache(tag, docker_registry) -> None:
+def load_docker_cache(platform, tag, docker_registry) -> None:
     """Imports tagged container from the given docker registry"""
     if docker_registry:
+        if platform in DOCKER_COMPOSE_WHITELIST:
+            env = os.environ.copy()
+            env["DOCKER_CACHE_REGISTRY"] = docker_registry
+            cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]
+            logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd))
+            check_call(cmd, env=env)
+            return
+
         # noinspection PyBroadException
         try:
             import docker_cache
@@ -398,7 +406,7 @@ def main() -> int:
         platform = args.platform
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
         if args.docker_registry and not args.no_pull:
-            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+            load_docker_cache(platform=platform, tag=tag, docker_registry=args.docker_registry)
         if not args.run_only:
             build_docker(platform=platform, registry=args.docker_registry, num_retries=args.docker_build_retries,
                          no_cache=args.no_cache, cache_intermediate=args.cache_intermediate)
@@ -442,7 +450,7 @@ def main() -> int:
         logging.info("Artifacts will be produced in the build/ directory.")
         for platform in platforms:
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-            load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+            load_docker_cache(platform=platform, tag=tag, docker_registry=args.docker_registry)
             build_docker(platform, registry=args.docker_registry, num_retries=args.docker_build_retries,
                          no_cache=args.no_cache)
             if args.build_only:

ci/docker/Dockerfile.build.centos7

@@ -119,22 +119,14 @@ RUN export SHORT_CUDA_VERSION=${CUDA_VERSION%.*} && \
         yum clean all; \
     fi
 
-# Python dependencies
-RUN pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir pylint cython numpy requests h5py scipy==1.2.3 wheel \
-    pytest==5.3.5 \
-    pytest-env==0.6.2 \
-    pytest-cov==2.8.1 \
-    pytest-xdist==1.31.0 \
-    pytest-timeout==1.3.4 \
-    mock==2.0.0 \
-    onnx==1.5.0 \
-    protobuf==3.5.2 \
-    tabulate==0.7.5
-
 # Fix the en_DK.UTF-8 locale to test locale invariance
 RUN localedef -i en_DK -f UTF-8 en_DK.UTF-8
 
+# Python dependencies
+RUN python3 -m pip install --upgrade pip
+COPY install/requirements /work/
+RUN python3 -m pip install -r /work/requirements
+
 ARG USER_ID=0
 COPY install/docker_filepermissions.sh /work/
 RUN /work/docker_filepermissions.sh

ci/docker/Dockerfile.build.jetson

@@ -20,7 +20,7 @@
 # This script assumes /work/mxnet exists and contains the mxnet code you wish to compile and
 # that /work/build exists and is the target for your output.
 
-FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
+FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04
 
 ENV ARCH=aarch64 \
     HOSTCC=gcc \
@@ -32,7 +32,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     build-essential \
     ninja-build \
     git \
-    curl \
+    wget \
     zip \
     unzip \
     python3 \
@@ -58,21 +58,38 @@ RUN git clone --recursive -b v0.3.9 https://github.com/xianyi/OpenBLAS.git && \
     cd /usr/local && \
     rm -rf OpenBLAS
 
-# Install aarch64 cross depedencies based on Jetpack 4.3
-# Manually downloaded using SDK Manager tool and placed in a private S3 bucket.
-# We're not allowed to redistribute these files and there is no public version.
-RUN aws s3 cp s3://mxnet-ci-prod-private-slave-data/nvidia/sdkm_downloads/cuda-repo-ubuntu1804-10-0-local-10.0.326-410.108_1.0-1_amd64.deb . && \
-    dpkg -i cuda-repo-ubuntu1804-10-0-local-10.0.326-410.108_1.0-1_amd64.deb && \
-    rm cuda-repo-ubuntu1804-10-0-local-10.0.326-410.108_1.0-1_amd64.deb && \
-    apt-key add /var/cuda-repo-10-0-local-10.0.326-410.108/7fa2af80.pub && \
-    aws s3 cp s3://mxnet-ci-prod-private-slave-data/nvidia/sdkm_downloads/cuda-repo-cross-aarch64-10-0-local-10.0.326_1.0-1_all.deb . && \
-    dpkg -i cuda-repo-cross-aarch64-10-0-local-10.0.326_1.0-1_all.deb && \
-    rm cuda-repo-cross-aarch64-10-0-local-10.0.326_1.0-1_all.deb && \
+# Install aarch64 cross depedencies based on Jetpack 4.4
+# Dependencies require cuda-toolkit-10.2 which isn't installed in nvidia docker container
+# It contains cuda-compat instead. However deb files currently depend on cuda-toolkit alone.
+# Hence force dpkg configure
+RUN wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cross-aarch64_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cudart-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cufft-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cupti-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-curand-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cusolver-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-cusparse-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-driver-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-misc-headers-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-npp-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nsight-compute-addon-l4t-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvgraph-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvml-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cuda/cuda-nvrtc-cross-aarch64-10-2_10.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/c/cublas/libcublas-cross-aarch64_10.2.2.89-1_all.deb && \
+    wget https://repo.download.nvidia.com/jetson/x86_64/pool/r32.4/n/nsight-compute/nsight-compute-addon-l4t-2019.5.0_2019.5.0.14-1_all.deb && \
+    dpkg -i --force-all  *.deb && \
+    rm *.deb && \
     apt-get update && \
     apt-get install -y -f && \
-    apt-get install -y cuda-cross-aarch64 cuda-cross-aarch64-10-0 && \
+    apt-get install -y cuda-cross-aarch64 cuda-cross-aarch64-10-2 && \
     rm -rf /var/lib/apt/lists/*
 
+# nvidia jetpack 4.4 installs libcublas.so at /usr/lib/aarch64-linux-gnu
+# while previously it used to store it at /usr/local/cuda/targets/aarch64-linux/lib/stubs
+RUN ln -s /usr/lib/aarch64-linux-gnu/libcublas.so /usr/local/cuda/targets/aarch64-linux/lib/stubs/libcublas.so
+
 ARG USER_ID=0
 ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/

ci/docker/install/requirements

@@ -22,6 +22,7 @@
 numpy>=1.17
 requests>=2.20.0,<3
 graphviz<0.9.0,>=0.8.1
+contextvars;python_version<"3.7"
 
 # Optional dependencies
 onnx==1.5.0
@@ -42,6 +43,7 @@ pytest-xdist==1.31.0
 pytest-timeout==1.3.4
 flaky==3.6.1
 setuptools
+wheel
 mock==2.0.0
 
 # TVM dependencies