Update CMake on CI

ci/build_windows.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 # Licensed to the Apache Software Foundation (ASF) under one
@@ -28,7 +28,9 @@
 import platform
 import shutil
 import sys
+import tempfile
 import time
+import zipfile
 from distutils.dir_util import copy_tree
 from enum import Enum
 from subprocess import check_call
@@ -147,22 +149,33 @@ def windows_build(args):
     mxnet_root = get_mxnet_root()
     logging.info("Found MXNet root: {}".format(mxnet_root))
 
-    with remember_cwd():
-        os.chdir(path)
-        cmd = "\"{}\" && cmake -G \"NMake Makefiles JOM\" {} {}".format(args.vcvars,
-                                                                        CMAKE_FLAGS[args.flavour],
-                                                                        mxnet_root)
-        logging.info("Generating project with CMake:\n{}".format(cmd))
-        check_call(cmd, shell=True)
-
-        cmd = "\"{}\" && jom".format(args.vcvars)
-        logging.info("Building with jom:\n{}".format(cmd))
-
-        t0 = int(time.time())
-        check_call(cmd, shell=True)
-
-        logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
-        logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+    url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
+    with tempfile.TemporaryDirectory() as tmpdir:
+        cmake_file_path = download_file(url, tmpdir)
+        with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
+            # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
+            zip_ref.extractall(tmpdir)
+
+        with remember_cwd():
+            os.chdir(path)
+            cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
+                args.vcvars,
+                os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
+                CMAKE_FLAGS[args.flavour], mxnet_root)
+            logging.info("Generating project with CMake:\n{}".format(cmd))
+            check_call(cmd, shell=True)
+
+            cmd = "\"{}\" && jom".format(args.vcvars)
+            logging.info("Building with jom:\n{}".format(cmd))
+
+            t0 = int(time.time())
+            check_call(cmd, shell=True)
+
+            logging.info(
+                "Build flavour: {} complete in directory: \"{}\"".format(
+                    args.flavour, os.path.abspath(path)))
+            logging.info("Build took {}".format(
+                datetime.timedelta(seconds=int(time.time() - t0))))
     windows_package(args)
 
 
@@ -262,4 +275,3 @@ def main():
 
 if __name__ == '__main__':
     sys.exit(main())
-

ci/docker/Dockerfile.build.android_armv7

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM mxnetcipinned/dockcross-base:11262018
+FROM dockcross/base
 MAINTAINER Pedro Larroy "[email protected]"
 
 # The cross-compiling emulator

ci/docker/Dockerfile.build.android_armv8

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARM64/ARMv8
 
-FROM mxnetcipinned/dockcross-base:11262018
+FROM dockcross/base
 MAINTAINER Pedro Larroy "[email protected]"
 
 RUN apt-get update && apt-get install -y \
@@ -82,4 +82,4 @@ RUN /work/ubuntu_adduser.sh
 
 COPY runtime_functions.sh /work/
 
-WORKDIR /work/build
+WORKDIR /work/build

ci/docker/Dockerfile.build.armv6

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for ARMv6
 
-FROM mxnetcipinned/dockcross-linux-armv6:11262018
+FROM dockcross/linux-armv6
 
 ENV ARCH armv6l
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.armv7

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for Android ARMv7
 
-FROM mxnetcipinned/dockcross-linux-armv7:11262018
+FROM dockcross/linux-armv7
 
 ENV ARCH armv7l
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.armv8

@@ -18,7 +18,7 @@
 #
 # Dockerfile to build MXNet for ARM64/ARMv8
 
-FROM mxnetcipinned/dockcross-linux-arm64:11262018
+FROM dockcross/linux-arm64
 
 ENV ARCH aarch64
 ENV HOSTCC gcc

ci/docker/Dockerfile.build.jetson

@@ -22,7 +22,7 @@
 
 FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
 
-FROM mxnetcipinned/dockcross-linux-arm64:11262018
+FROM dockcross/linux-arm64
 
 ENV ARCH aarch64
 ENV HOSTCC gcc

ci/docker/install/ubuntu_core.sh

@@ -57,11 +57,10 @@ apt-get install -y \
 ln -s /usr/lib/x86_64-linux-gnu/libturbojpeg.so.0.1.0 /usr/lib/x86_64-linux-gnu/libturbojpeg.so
 
 
-# Note: we specify an exact cmake version to work around a cmake 3.10 CUDA 10 issue.
-# Reference: https://github.com/clab/dynet/issues/1457
+# CMake 3.13.2+ is required
 mkdir /opt/cmake && cd /opt/cmake
-wget -nv https://cmake.org/files/v3.12/cmake-3.12.4-Linux-x86_64.sh
-sh cmake-3.12.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
+wget -nv https://cmake.org/files/v3.13/cmake-3.13.5-Linux-x86_64.sh
+sh cmake-3.13.5-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
 ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
-rm cmake-3.12.4-Linux-x86_64.sh
+rm cmake-3.13.5-Linux-x86_64.sh
 cmake --version

ci/util.py

@@ -15,12 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
 import contextlib
 import logging
 import logging.config
+import os
+import subprocess
 import sys
 
+import requests
+
 
 def get_mxnet_root() -> str:
     curpath = os.path.abspath(os.path.dirname(__file__))
@@ -130,3 +133,31 @@ def config_logging():
     # or sensitive information
     logging.getLogger("botocore").setLevel(logging.WARNING)
     logging.getLogger("requests").setLevel(logging.WARNING)
+
+
+# Takes url and downloads it to the dest_path directory on Windows.
+def download_file(url, dest_path):
+    file_name = url.split('/')[-1]
+    full_path = "{}\\{}".format(dest_path, file_name)
+    logging.info("Downloading: {}".format(full_path))
+    r = requests.get(url, stream=True)
+    if r.status_code == 404:
+        return r.status_code
+    elif r.status_code != 200:
+        logging.error("{} returned status code {}".format(url, r.status_code))
+    with open(full_path, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024):
+            if chunk: # filter out keep-alive new chunks
+                f.write(chunk)
+    return full_path
+
+
+# Takes arguments and runs command on host.  Shell is disabled by default.
+def run_command(args, shell=False):
+    try:
+        logging.info("Issuing command: {}".format(args))
+        res = subprocess.check_output(args, shell=shell, timeout=1800).decode("utf-8").replace("\r\n", "")
+        logging.info("Output: {}".format(res))
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
+    return res