Docker build script robustness (apache#2710)

* [DOCKER] Make all install .sh scripts directly executable.

* [DOCKER] Use curl -L consistently.

Make the use of the curl -L option in docker build scripts consistent.

* [DOCKER] Drop use of --force-yes

The --force-yes option is generally not recommend, it can leave
systems in an undefined state.  The use of --allow-* options is
preferred.  In this particular case the --force-yes option appears to
serve no purpose.  Dropping it.

* [DOCKER] Drop superflous repeated apt-get update.

The "apt-get update && apt-get install" idiom is necessary and
specific to Dockerfile.  In shell the repeated apt-get update is
superflous.  Drop the duplicates.

* [DOCKER] Robustness -e -u -o pipefail

The install scripts used to construct docker environments do not, in
general, propagate errors.  Some of the scripts use adhoc &&
directives to chain together short sequences of commands but there are
numerous failure modes which are silently ignored.  This patch puts in
place some consistent, basic, shell error trapping across all of the
install scripts.

Note this is a step forward towards more robust scripts but it is not
a complete solution.

* [DOCKER] Shallow clone.

Use shallow clone to reduce bandwidth requirements of repeated docker
(re)-builds.

* [DOCKER] Use clone --branch rather than clone then checkout

Use the git clone --branch idiom rather than git clone && git
checkout.  This paves the way for using --depth=1

docker/Dockerfile.ci_gpu

@@ -24,7 +24,7 @@ COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
 RUN bash /install/ubuntu_install_sphinx.sh
 
 # Fix recommonmark to latest version
-RUN git clone https://github.com/rtfd/recommonmark
+RUN git clone --depth=1 https://github.com/rtfd/recommonmark
 RUN cd recommonmark; python3 setup.py install
 
 # Enable doxygen for c++ doc build

docker/Dockerfile.demo_opencl

@@ -45,7 +45,7 @@ RUN echo "Cloning TVM source & submodules"
 ENV TVM_PAR_DIR="/usr"
 RUN mkdir -p TVM_PAR_DIR && \
 	cd ${TVM_PAR_DIR} && \
-	git clone https://github.com/dmlc/tvm --recursive
+	git clone --depth=1 https://github.com/dmlc/tvm --recursive
 #RUN git submodule update --init --recursive
 
 

docker/install/install_tvm_cpu.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr
-git clone https://github.com/dmlc/tvm --recursive
+git clone --depth=1 https://github.com/dmlc/tvm --recursive
 cd /usr/tvm
 echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake
 echo set\(USE_RPC ON\) >> config.cmake

docker/install/install_tvm_gpu.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr
-git clone https://github.com/dmlc/tvm --recursive
+git clone --depth=1 https://github.com/dmlc/tvm --recursive
 cd /usr/tvm
 echo set\(USE_LLVM llvm-config-6.0\) >> config.cmake
 echo set\(USE_CUDA ON\) >> config.cmake

docker/install/ubuntu_install_androidsdk.sh

@@ -1,6 +1,9 @@
+#!/bin/bash
+
 . /etc/profile
 
 set -o errexit -o nounset
+set -o pipefail
 
 ANDROID_HOME=/opt/android-sdk-linux
 ASDKTOOLS_HOME=/opt/android-sdk-tools
@@ -58,7 +61,7 @@ EOF
 
 mkdir /root/.android 2>/dev/null || true
 touch /root/.android/repositories.cfg
-yes | sdkmanager --licenses --sdk_root="$ANDROID_HOME"
+(yes || true) | sdkmanager --licenses --sdk_root="$ANDROID_HOME"
 sdkmanager --verbose --package_file=/install/package-list-minimal.txt --sdk_root="$ANDROID_HOME"
 test -d "${ANDROID_HOME}/build-tools/27.0.3"
 test -d "${ANDROID_HOME}/ndk-bundle"

docker/install/ubuntu_install_antlr.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 cd /usr/local/lib
 wget -q https://www.antlr.org/download/antlr-4.7.1-complete.jar
 cd -

docker/install/ubuntu_install_caffe2.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 python3 -m caffe2.python.models.download -i -f squeezenet
 python3 -m caffe2.python.models.download -i -f resnet50
 python3 -m caffe2.python.models.download -i -f vgg19

docker/install/ubuntu_install_core.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install libraries for building c++ core on ubuntu
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+apt-get update && apt-get install -y --no-install-recommends \
         git make libgtest-dev cmake wget unzip libtinfo-dev libz-dev\
         libcurl4-openssl-dev libopenblas-dev g++ sudo
 

docker/install/ubuntu_install_coreml.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install coremltools

docker/install/ubuntu_install_darknet.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 #install the necessary dependancies, cffi, opencv
 wget -q 'https://github.com/siju-samuel/darknet/blob/master/lib/libdarknet.so?raw=true' -O libdarknet.so
 pip2 install opencv-python cffi

docker/install/ubuntu_install_emscripten.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 alias make="make -j4"
 
 # Get latest cmake

docker/install/ubuntu_install_gluoncv.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install gluoncv

docker/install/ubuntu_install_golang.sh

@@ -1,4 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 #install the necessary dependancies for golang build
-apt-get update && apt-get install -y golang-1.10-go
-apt-get update && apt-get install -y golang-1.10-doc
-apt-get update && apt-get install -y golint
+apt-get update
+apt-get install -y golang-1.10-go
+apt-get install -y golang-1.10-doc
+apt-get install -y golint

docker/install/ubuntu_install_gradle.sh

@@ -1,6 +1,9 @@
+#!/bin/bash
+
 . /etc/profile
 
 set -o errexit -o nounset
+set -o pipefail
 
 GRADLE_HOME=/opt/gradle
 GRADLE_VERSION=4.10-rc-2

docker/install/ubuntu_install_iverilog.sh

@@ -1,4 +1,10 @@
-apt-get install -y --no-install-recommends --force-yes make bison flex
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get install -y --no-install-recommends make bison flex
 wget -q ftp://icarus.com/pub/eda/verilog/v10/verilog-10.1.tar.gz
 tar xf verilog-10.1.tar.gz
 cd verilog-10.1

docker/install/ubuntu_install_java.sh

@@ -1,4 +1,8 @@
+#!/bin/bash
+
 set -o errexit -o nounset
+set -o pipefail
+
 apt-get update && apt-get install -y openjdk-8-jdk maven
 test -d "/usr/lib/jvm/java-8-openjdk-amd64/jre"
 echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /etc/profile

docker/install/ubuntu_install_keras.sh

@@ -1,2 +1,8 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip2 install keras tensorflow h5py
 pip3 install keras tensorflow h5py

docker/install/ubuntu_install_llvm.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\
      >> /etc/apt/sources.list.d/llvm.list
 echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\
@@ -19,4 +25,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0
+apt-get update && apt-get install -y llvm-4.0 llvm-5.0 llvm-6.0 clang-6.0

docker/install/ubuntu_install_mxnet.sh

@@ -1 +1,7 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 pip3 install mxnet

docker/install/ubuntu_install_nnpack.sh

@@ -1,11 +1,13 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes git cmake
+#!/bin/bash
 
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends git cmake
 
-git clone https://github.com/Maratyszcza/NNPACK NNPACK
-cd NNPACK
 # TODO: specific tag?
-git checkout 1e005b0c2
-cd -
+git clone --branch=1e005b0c2 --depth=1 https://github.com/Maratyszcza/NNPACK NNPACK
 
 mkdir -p NNPACK/build
 cd NNPACK/build

docker/install/ubuntu_install_nodejs.sh

@@ -1,4 +1,16 @@
-apt-get update && apt-get install -y curl
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update
+apt-get install -y curl
+
+# The node install script fetched and executed here will update the
+# apt source list, hence the second apt-get update is necessary.
 curl -s -S -L https://deb.nodesource.com/setup_6.x | bash -
-apt-get update && apt-get install -y nodejs
+apt-get update
+apt-get install -y nodejs
+
 npm install eslint jsdoc ws

docker/install/ubuntu_install_onnx.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # fix to certain version for now
 pip2 install onnx>=1.1.0
 pip3 install onnx>=1.1.0

docker/install/ubuntu_install_opencl.sh

@@ -1,5 +1,11 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # Install OpenCL runtime in nvidia docker.
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+apt-get update && apt-get install -y --no-install-recommends \
         ocl-icd-opencl-dev \
         clinfo && \
     rm -rf /var/lib/apt/lists/*

docker/install/ubuntu_install_opengl.sh

@@ -1,4 +1,10 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 apt-get update --fix-missing
 
-apt-get install -y --no-install-recommends --force-yes \
-        libgl1-mesa-dev libglfw3-dev
+apt-get install -y --no-install-recommends \
+        libgl1-mesa-dev libglfw3-dev

docker/install/ubuntu_install_python.sh

@@ -1,10 +1,19 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install python and pip, don't modify this, modify install_python_package.sh
-apt-get update && apt-get install -y python-dev
+apt-get update
+apt-get install -y python-dev
 
 # python 3.6
-apt-get update && apt-get install -y software-properties-common
-add-apt-repository ppa:jonathonf/python-3.6 &&\
-    apt-get update && apt-get install -y python-pip python-dev python3.6 python3.6-dev
+apt-get install -y software-properties-common
+
+add-apt-repository ppa:jonathonf/python-3.6
+apt-get update
+apt-get install -y python-pip python-dev python3.6 python3.6-dev
 
 rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
 

docker/install/ubuntu_install_python_package.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # install libraries for python package on ubuntu
 pip2 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typing antlr4-python2-runtime attrs
 pip3 install nose pylint==2.2.2 six numpy nose-timer cython decorator scipy tornado typed_ast pytest mypy orderedset antlr4-python3-runtime attrs

docker/install/ubuntu_install_redis.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 apt-get update && apt-get install -y redis-server
 pip2 install xgboost psutil
 pip3 install xgboost psutil

docker/install/ubuntu_install_rocm.sh

@@ -1,4 +1,10 @@
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
 # Install ROCm cross compilation toolchain.
 wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add -
 echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main > /etc/apt/sources.list.d/rocm.list
-apt-get update && apt-get install -y --force-yes rocm-dev
+apt-get update && apt-get install -y rocm-dev

docker/install/ubuntu_install_rust.sh

@@ -1,9 +1,15 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes curl
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends curl
 
 export RUSTUP_HOME=/opt/rust
 export CARGO_HOME=/opt/rust
 # this rustc is one supported by the installed version of rust-sgx-sdk
-curl -s -S https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28
+curl -s -S -L https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain nightly-2019-01-28
 . $CARGO_HOME/env
 rustup toolchain add nightly
 rustup component add rust-src

docker/install/ubuntu_install_sgx.sh

@@ -1,21 +1,26 @@
-apt-get update && apt-get install -y --no-install-recommends --force-yes \
+#!/bin/bash
+
+set -e
+set -u
+set -o pipefail
+
+apt-get update && apt-get install -y --no-install-recommends \
     build-essential git cmake \
     wget python pkg-config software-properties-common \
     autoconf automake libtool ocaml \
     protobuf-compiler libprotobuf-dev \
     libssl-dev libcurl4-openssl-dev curl
 
-git clone https://github.com/intel/linux-sgx.git
+git clone --branch=sgx_2.2 --depth=1 https://github.com/intel/linux-sgx.git
 cd linux-sgx
-git checkout sgx_2.2
-curl -s -S 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply
+curl -s -S -L 'https://gist.githubusercontent.com/nhynes/c770b0e91610f8c020a8d1a803a1e7cb/raw/8f5372d9cb88929b3cc49a384943bb363bc06827/intel-sgx.patch' | git apply
 ./download_prebuilt.sh
 make -j4 sdk && make -j4 sdk_install_pkg
 ./linux/installer/bin/sgx_linux_x64_sdk*.bin --prefix /opt
 cd -
 
-git clone https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk
+tag=6098af # v1.0.5
+git clone --branch=$tag --depth=1 https://github.com/baidu/rust-sgx-sdk.git /opt/rust-sgx-sdk
 cd /opt/rust-sgx-sdk
-git checkout 6098af # v1.0.5
-curl -s -S 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply
+curl -s -S -L 'https://gist.githubusercontent.com/nhynes/37164039c5d3f33aa4f123e4ba720036/raw/b0de575fe937231799930764e76c664b92975163/rust-sgx-sdk.diff' | git apply
 cd -