Revert "Revert "CI scripts refinements. Separate Py2 and Py3 installs…

… cripts. Fix perms. (apache#12125)" (apache#12223)"

This reverts commit 2ff1837.

add docs_requirements

ci/build.py

@@ -43,6 +43,43 @@
 
 CCACHE_MAXSIZE = '500G'
 
+
+
+def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay_s: initial delay between retries in seconds
+    :type delay_s: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    """
+    import time
+    from functools import wraps
+    def decorated_retry(f):
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay_s
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+        return f_retry  # true decorator
+    return decorated_retry
+
 def under_ci() -> bool:
     """:return: True if we run in Jenkins."""
     return 'JOB_NAME' in os.environ
@@ -77,9 +114,8 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
     :param num_retries: Number of retries to build the docker image
     :return: Id of the top level image
     """
-
     tag = get_docker_tag(platform=platform, registry=registry)
-    logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+    logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
     #
     # We add a user with the same group as the executing non-root user so files created in the
     # container match permissions of the local user. Same for the group.
@@ -91,40 +127,24 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
     # docker pull see: docker_cache.load_docker_cache
     #
     # This doesn't work with multi head docker files.
-    # 
-
-    for i in range(num_retries):
-        logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
-
-        cmd = [docker_binary, "build",
-               "-f", get_dockerfile(platform),
-               "--build-arg", "USER_ID={}".format(os.getuid()),
-               "--build-arg", "GROUP_ID={}".format(os.getgid()),
-               "--cache-from", tag,
-               "-t", tag,
-               "docker"]
+    #
+    cmd = [docker_binary, "build",
+           "-f", get_dockerfile(platform),
+           "--build-arg", "USER_ID={}".format(os.getuid()),
+           "--build-arg", "GROUP_ID={}".format(os.getgid()),
+           "--cache-from", tag,
+           "-t", tag,
+           "docker"]
+
+    @retry(subprocess.CalledProcessError, tries=num_retries)
+    def run_cmd():
         logging.info("Running command: '%s'", ' '.join(cmd))
-        try:
-            check_call(cmd)
-            # Docker build was successful. Call break to break out of the retry mechanism
-            break
-        except subprocess.CalledProcessError as e:
-            saved_exception = e
-            logging.error('Failed to build docker image')
-            # Building the docker image failed. Call continue to trigger the retry mechanism
-            continue
-    else:
-        # Num retries exceeded
-        logging.exception('Exception during build of docker image', saved_exception)
-        logging.fatal('Failed to build the docker image, aborting...')
-        sys.exit(1)
+        check_call(cmd)
 
+    run_cmd()
     # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
     # check_call would have failed
-    image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
-    if not image_id:
-        raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
-    return image_id
+    return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
 
 
 def _get_local_image_id(docker_binary, docker_tag):
@@ -136,6 +156,8 @@ def _get_local_image_id(docker_binary, docker_tag):
     cmd = [docker_binary, "images", "-q", docker_tag]
     image_id_b = subprocess.check_output(cmd)
     image_id = image_id_b.decode('utf-8').strip()
+    if not image_id:
+        raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
     return image_id
 
 
@@ -186,7 +208,7 @@ def container_run(platform: str,
                '-e', "CCACHE_LOGFILE=/tmp/ccache.log",  # a container-scoped log, useful for ccache verification.
                tag]
     runlist.extend(command)
-    cmd = '\\\n\t'.join(runlist)
+    cmd = ' \\\n\t'.join(runlist)
     ret = 0
     if not dry_run and not interactive:
         logging.info("Running %s in container %s", command, tag)
@@ -199,14 +221,14 @@ def container_run(platform: str,
         # -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
         idx = into_cmd.index('-u') + 2
         into_cmd[idx:idx] = ['-ti']
-        cmd = '\\\n\t'.join(into_cmd)
+        cmd = ' \\\n\t'.join(into_cmd)
         logging.info("Executing:\n%s\n", cmd)
         docker_run_cmd = ' '.join(into_cmd)
         ret = call(into_cmd)
 
     if not dry_run and not interactive and ret != 0:
         logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
-        logging.error("You can get into the container by adding the -i option")
+        logging.error("You can get into the container by adding the -i option to this script")
         raise subprocess.CalledProcessError(ret, cmd)
 
     return docker_run_cmd
@@ -303,7 +325,6 @@ def use_cache():
     command = list(chain(*args.command))
     docker_binary = get_docker_binary(args.nvidiadocker)
     shared_memory_size = args.shared_memory_size
-    num_docker_build_retires = args.docker_build_retries
 
     if args.list:
         list_platforms()
@@ -312,7 +333,7 @@ def use_cache():
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
         if use_cache():
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-        build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
+        build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
         if args.build_only:
             logging.warning("Container was just built. Exiting due to build-only.")
             return 0
@@ -346,7 +367,7 @@ def use_cache():
             tag = get_docker_tag(platform=platform, registry=args.docker_registry)
             if use_cache():
                 load_docker_cache(tag=tag, docker_registry=args.docker_registry)
-            build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
+            build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
             if args.build_only:
                 continue
             build_platform = "build_{}".format(platform)

ci/docker/Dockerfile.build.ubuntu_blc

@@ -24,8 +24,13 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/ubuntu_npm_blc.sh /work/
 RUN /work/ubuntu_npm_blc.sh
 

ci/docker/Dockerfile.build.ubuntu_build_cuda

@@ -27,20 +27,30 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/
 RUN /work/ubuntu_scala.sh
+
 COPY install/ubuntu_r.sh /work/
 COPY install/r.gpg /work/
 RUN /work/ubuntu_r.sh
+
 COPY install/ubuntu_perl.sh /work/
 RUN /work/ubuntu_perl.sh
+
 COPY install/ubuntu_clang.sh /work/
 RUN /work/ubuntu_clang.sh
+
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
 

ci/docker/Dockerfile.build.ubuntu_cpu

@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_gpu

@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt

@@ -24,10 +24,16 @@ WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
 RUN /work/ubuntu_core.sh
+
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
 COPY install/tensorrt.sh /work
 RUN /work/tensorrt.sh
 

ci/docker/Dockerfile.build.ubuntu_nightly_cpu

@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/Dockerfile.build.ubuntu_nightly_gpu

@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
 COPY install/deb_ubuntu_ccache.sh /work/
 RUN /work/deb_ubuntu_ccache.sh
 
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
 
 COPY install/ubuntu_scala.sh /work/
 COPY install/sbt.gpg /work/

ci/docker/install/ubuntu_python.sh → ci/docker/install/ubuntu_python2.sh

@@ -22,12 +22,10 @@
 
 set -ex
 # install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev virtualenv wget
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
 wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
 python2 get-pip.py
 
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock

ci/docker/install/ubuntu_python3.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+# install libraries for mxnet's python package on ubuntu
+apt-get install -y python3-dev virtualenv wget
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock

ci/docker/runtime_functions.sh

@@ -995,6 +995,7 @@ broken_link_checker() {
     ./tests/nightly/broken_link_checker_test/broken_link_checker.sh
 }
 
+
 ##############################################################
 # MAIN
 #