[v1.x] Backport #20782, #20783 and #20786 from v1.9.x branch, #19945 from master

ci/build.py

@@ -34,6 +34,7 @@
 import shutil
 import signal
 import subprocess
+from platform import machine
 from itertools import chain
 from subprocess import check_call, check_output
 from typing import *
@@ -46,7 +47,7 @@
 DOCKER_COMPOSE_FILES = set(['docker/build.centos7'])
 
 # keywords to identify arm-based dockerfiles
-AARCH_FILE_KEYWORDS = ['armv', 'aarch64']
+AARCH_FILE_KEYWORDS = ['aarch64']
 
 def get_dockerfiles_path():
     return "docker"
@@ -60,15 +61,15 @@ def get_docker_compose_platforms(path: str = get_dockerfiles_path()):
     return platforms
 
 
-def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]:
+def get_platforms(path: str = get_dockerfiles_path(), arch=machine()) -> List[str]:
     """Get a list of platforms given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
     dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles))
     files = set(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
     files = files - DOCKER_COMPOSE_FILES
     files.update(["build."+x for x in get_docker_compose_platforms()])
     arm_files = set(filter(lambda x: any(y in x for y in AARCH_FILE_KEYWORDS), files))
-    if arch == 'x86':
+    if arch == 'x86_64':
         files = files - arm_files
     elif arch == 'aarch64':
         files = arm_files
@@ -273,15 +274,10 @@ def container_run(platform: str,
         # mount mxnet/build for storing build
         '-v', "{}:/work/build".format(local_build_folder),
         '-v', "{}:/work/ccache".format(local_ccache_dir),
-        '-u', '{}:{}'.format(os.getuid(), os.getgid()),
-        '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),
-        # temp dir should be local and not shared
-        '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),
-        # this path is inside the container as /work/ccache is mounted
-        '-e', "CCACHE_DIR={}".format(environment['CCACHE_DIR']),
-        # a container-scoped log, useful for ccache verification.
-        '-e', "CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']),
+        '-u', '{}:{}'.format(os.getuid(), os.getgid())
     ]
+    for e in environment.keys():
+        docker_arg_list += ['-e', '{}={}'.format(e, environment[e])]
     docker_arg_list += [tag]
     docker_arg_list.extend(command)
 
@@ -305,7 +301,7 @@ def docker_run_cmd(cmd):
     return 0
 
 
-def list_platforms(arch='x86') -> str:
+def list_platforms(arch=machine()) -> str:
     return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch)))
 
 
@@ -362,8 +358,8 @@ def main() -> int:
                         type=str)
 
     parser.add_argument("-A", "--architecture",
-                        help="Architecture of images to build (x86 or aarch64). Default is x86.",
-                        default='x86',
+                        help="Architecture of images to build (x86_64 or aarch64). Default is current machine type.",
+                        default=machine(),
                         dest='architecture')
 
     parser.add_argument("-b", "--build-only",

ci/docker/install/requirements_aarch64

@@ -30,3 +30,4 @@ astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 setuptools
 coverage
+wheel

ci/docker_cache.py

@@ -30,6 +30,7 @@
 import subprocess
 import re
 import sys
+from platform import machine
 from typing import *
 
 import build as build_util
@@ -225,7 +226,7 @@ def script_name() -> str:
 
     args = parser.parse_args()
 
-    platforms = build_util.get_platforms()
+    platforms = build_util.get_platforms(arch=machine())
 
     if "dkr.ecr" in args.docker_registry:
         _ecr_login(args.docker_registry)

cpp-package/example/get_data.sh

@@ -33,11 +33,10 @@ download () {
     fi
 
     echo "Downloading ${URL} ..."
-    local CURL_OPTIONS="--connect-timeout 10 \
+    local CURL_OPTIONS="--connect-timeout 60 \
               --max-time 300 \
-              --retry-delay 10 \
-              --retry 3 \
-              --retry-delay 0 \
+              --retry-delay 30 \
+              --retry 5 \
               --location \
               --silent"
     curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}

docs/python_docs/themes/mx-theme/mxtheme/header_top.html

@@ -1,6 +1,6 @@
 <header class="site-header" role="banner">
   <div class="wrapper">
-      <a class="site-title" rel="author" href="/versions/1.8.0/"><img
+      <a class="site-title" rel="author" href="/versions/1.9.0/"><img
             src="{{pathto('_static/mxnet_logo.png', 1)}}" class="site-header-logo"></a>
     <nav class="site-nav">
       <input type="checkbox" id="nav-trigger" class="nav-trigger"/>
@@ -13,19 +13,20 @@
       </label>
 
       <div class="trigger">
-        <a class="page-link" href="/versions/1.8.0/get_started">Get Started</a>
-        <a class="page-link" href="/versions/1.8.0/blog">Blog</a>
-        <a class="page-link" href="/versions/1.8.0/features">Features</a>
-        <a class="page-link" href="/versions/1.8.0/ecosystem">Ecosystem</a>
-        <a class="page-link page-current" href="/versions/1.8.0/api">Docs & Tutorials</a>
+        <a class="page-link" href="/versions/1.9.0/get_started">Get Started</a>
+        <a class="page-link" href="/versions/1.9.0/blog">Blog</a>
+        <a class="page-link" href="/versions/1.9.0/features">Features</a>
+        <a class="page-link" href="/versions/1.9.0/ecosystem">Ecosystem</a>
+        <a class="page-link page-current" href="/versions/1.9.0/api">Docs & Tutorials</a>
         <a class="page-link" href="https://github.com/apache/incubator-mxnet">GitHub</a>
         <div class="dropdown">
-          <span class="dropdown-header">1.8.0
+          <span class="dropdown-header">1.9.0
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a class="dropdown-option" href="/">master</a><br>
-            <a class="dropdown-option-active" href="/versions/1.8.0/">1.8.0</a><br>
+            <a class="dropdown-option-active" href="/versions/1.9.0/">1.9.0</a><br>
+            <a class="dropdown-option" href="/versions/1.8.0/">1.8.0</a><br>
             <a class="dropdown-option" href="/versions/1.7.0/">1.7.0</a><br>
             <a class="dropdown-option" href="/versions/1.6.0/">1.6.0</a><br>
             <a class="dropdown-option" href="/versions/1.5.0/">1.5.0</a><br>

docs/static_site/src/404.html

@@ -1,23 +1,19 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
 ---
+  # Licensed to the Apache Software Foundation (ASF) under one
+  # or more contributor license agreements.  See the NOTICE file
+  # distributed with this work for additional information
+  # regarding copyright ownership.  The ASF licenses this file
+  # to you under the Apache License, Version 2.0 (the
+  # "License"); you may not use this file except in compliance
+  # with the License.  You may obtain a copy of the License at
+  #   http://www.apache.org/licenses/LICENSE-2.0
+  # Unless required by applicable law or agreed to in writing,
+  # software distributed under the License is distributed on an
+  # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  # KIND, either express or implied.  See the License for the
+  # specific language governing permissions and limitations
+  # under the License.
+
 layout: default
 ---
 

docs/static_site/src/_config.yml

@@ -37,9 +37,10 @@ description: >- # this means to ignore newlines until "baseurl:"
 twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_config_beta.yml

@@ -39,9 +39,10 @@ url: https://thomasdelteil.github.io
 twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_config_prod.yml

@@ -39,9 +39,10 @@ twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
 google_analytics: UA-96378503-1
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_includes/get_started/get_started.html

@@ -1,7 +1,7 @@
 <script>
     /** Defaults **/
     /** See options.js for the full ugly script **/
-    var versionSelect = defaultVersion = 'v1.8.0';
+    var versionSelect = defaultVersion = 'v1.9.0';
     var platformSelect = 'linux';
     var languageSelect = 'python';
     var processorSelect = 'cpu';
@@ -24,13 +24,14 @@ <h2>Platform and use-case specific instructions for using MXNet</h2>
             <div class="col-9 install-right">
                 <div class="dropdown" id="version-dropdown-container">
                     <button class="current-version dropbtn btn" type="button" data-toggle="dropdown">
-                        v1.8.0
+                        v1.9.0
                         <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true">
                             <path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path>
                         </svg>
                     </button>
                     <ul class="opt-group version-dropdown">
-                        <li class="opt active versions"><a href="#">v1.8.0</a></li>
+                        <li class="opt active versions"><a href="#">v1.9.0</a></li>
+                        <li class="opt versions"><a href="#">v1.8.0</a></li>
                         <li class="opt versions"><a href="#">v1.7.0</a></li>
                         <li class="opt versions"><a href="#">v1.6.0</a></li>
                         <li class="opt versions"><a href="#">v1.5.1</a></li>

docs/static_site/src/_includes/get_started/linux/python/cpu/pip.md

@@ -9,11 +9,18 @@ page](https://mxnet.apache.org/get_started/download).
 
 Run the following command:
 
-<div class="v1-8-0">
+<div class="v1-9-0">
 {% highlight bash %}
 pip install mxnet
 {% endhighlight %}
 
+</div> <!-- End of v1-9-0 -->
+
+<div class="v1-8-0">
+{% highlight bash %}
+pip install mxnet==1.8.0.post0
+{% endhighlight %}
+
 Start from 1.7.0 release, oneDNN(previously known as: MKL-DNN/DNNL) is enabled
 in pip packages by default.
 
@@ -35,7 +42,7 @@ MXNet tuning guide</a>.
 To install native MXNet without oneDNN, run the following command:
 
 {% highlight bash %}
-pip install mxnet-native
+pip install mxnet-native==1.8.0.post0
 {% endhighlight %}
 
 </div> <!-- End of v1-8-0 -->

docs/static_site/src/_includes/get_started/linux/python/gpu/pip.md

@@ -10,11 +10,18 @@ page](https://mxnet.apache.org/get_started/download).
 
 Run the following command:
 
-<div class="v1-8-0">
+<div class="v1-9-0">
 {% highlight bash %}
 $ pip install mxnet-cu102
 {% endhighlight %}
 
+</div> <!-- End of v1-9-0 -->
+
+<div class="v1-8-0">
+{% highlight bash %}
+$ pip install mxnet-cu102==1.8.0.post0
+{% endhighlight %}
+
 </div> <!-- End of v1-8-0 -->
 
 <div class="v1-7-0">

docs/static_site/src/_includes/get_started/pip_snippet.md

@@ -1,7 +1,7 @@
 You can then <a href="/get_started/validate_mxnet.html">validate your MXNet installation</a>.
 
 <div style="text-align: center">
-    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.8.0.png"
+    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.9.0.png"
     alt="pip packages"/>
 </div>
 

docs/static_site/src/_includes/header.html

@@ -87,13 +87,14 @@
         <a class="page-link" href="{{'/api' | relative_url }}">Docs & Tutorials</a>
         <a class="page-link" href="https://github.com/apache/incubator-mxnet">GitHub</a>
         <div class="dropdown">
-          <span class="dropdown-header">1.8.0
+          <span class="dropdown-header">1.9.0
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a href="/">master</a>
-            <a class="dropdown-option-active" href="/versions/1.8.0/">1.8.0</a>
-            <a href="/versions/1.6.0/">1.7.0</a>
+            <a class="dropdown-option-active" href="/versions/1.9.0/">1.9.0</a>
+            <a href="/versions/1.8.0/">1.8.0</a>
+            <a href="/versions/1.7.0/">1.7.0</a>
             <a href="/versions/1.6.0/">1.6.0</a>
             <a href="/versions/1.5.0/">1.5.0</a>
             <a href="/versions/1.4.1/">1.4.1</a>

docs/static_site/src/_layouts/default.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+
 <!---
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
@@ -7,16 +8,15 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   KIND, either express or implied.  See the License for the
   specific language governing permissions and limitations
   under the License.
 -->
+
 <html lang="{{ page.lang | default: site.lang | default: " en" }}">
 
 {%- include head.html -%}

docs/static_site/src/_layouts/home.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+
 <!---
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
@@ -7,16 +8,15 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   KIND, either express or implied.  See the License for the
   specific language governing permissions and limitations
   under the License.
 -->
+
 <html lang="{{ page.lang | default: site.lang | default: " en" }}">
 
 {%- include head.html -%}