From 909adece0047535e5b3db2badd311299d113539f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 15 Oct 2020 10:41:27 +0100 Subject: [PATCH] [ML] Retry pulling of Docker images to work around network issues (#1535) Adds a function to retry pulling Docker images up to 5 times, and calls it to pre-fetch every Docker image that's used as a "FROM" image by our "docker build" commands. "docker build" will pull the "FROM" image if it's not present on the local machine, but will only try once, so is vulnerable to a transient problem with the Docker registry failing the whole build. By retrying "docker pull" several times before running "docker build" we ensure that the image is present locally when "docker build" runs and hence "docker build" never needs to pull it. --- dev-tools/docker/build_check_style_image.sh | 7 +++- .../build_linux_aarch64_cross_build_image.sh | 7 +++- .../build_linux_aarch64_native_build_image.sh | 7 +++- dev-tools/docker/build_linux_build_image.sh | 7 +++- dev-tools/docker/build_macosx_build_image.sh | 7 +++- dev-tools/docker/prefetch_docker_image.sh | 38 +++++++++++++++++++ dev-tools/docker_build.sh | 3 ++ dev-tools/docker_check_style.sh | 2 + dev-tools/docker_test.sh | 3 ++ dev-tools/jenkins_ci.sh | 3 ++ 10 files changed, 74 insertions(+), 10 deletions(-) create mode 100755 dev-tools/docker/prefetch_docker_image.sh diff --git a/dev-tools/docker/build_check_style_image.sh b/dev-tools/docker/build_check_style_image.sh index 257d166f11..0f483315f2 100755 --- a/dev-tools/docker/build_check_style_image.sh +++ b/dev-tools/docker/build_check_style_image.sh @@ -21,9 +21,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION check_style_image +. ./prefetch_docker_image.sh +CONTEXT=check_style_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh index 74f5a6389c..d92b96a5eb 100755 --- a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh +++ b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh @@ -23,9 +23,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_cross_image +. ./prefetch_docker_image.sh +CONTEXT=linux_aarch64_cross_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_aarch64_native_build_image.sh b/dev-tools/docker/build_linux_aarch64_native_build_image.sh index dc38701280..6829d3c117 100755 --- a/dev-tools/docker/build_linux_aarch64_native_build_image.sh +++ b/dev-tools/docker/build_linux_aarch64_native_build_image.sh @@ -29,9 +29,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_aarch64_native_image +. ./prefetch_docker_image.sh +CONTEXT=linux_aarch64_native_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_linux_build_image.sh b/dev-tools/docker/build_linux_build_image.sh index 8bea9b556f..2922610a45 100755 --- a/dev-tools/docker/build_linux_build_image.sh +++ b/dev-tools/docker/build_linux_build_image.sh @@ -29,9 +29,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION linux_image +. ./prefetch_docker_image.sh +CONTEXT=linux_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/build_macosx_build_image.sh b/dev-tools/docker/build_macosx_build_image.sh index 69392663a0..59e783817a 100755 --- a/dev-tools/docker/build_macosx_build_image.sh +++ b/dev-tools/docker/build_macosx_build_image.sh @@ -23,9 +23,12 @@ set -e cd `dirname $0` -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION macosx_image +. ./prefetch_docker_image.sh +CONTEXT=macosx_image +prefetch_docker_image $CONTEXT/Dockerfile +docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT # Get a username and password for this by visiting -# https://docker.elastic.co:7000 and allowing it to authenticate against your +# https://docker-auth.elastic.co and allowing it to authenticate against your # GitHub account docker login $HOST docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION diff --git a/dev-tools/docker/prefetch_docker_image.sh b/dev-tools/docker/prefetch_docker_image.sh new file mode 100755 index 0000000000..aa315104a0 --- /dev/null +++ b/dev-tools/docker/prefetch_docker_image.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# Pull a docker image, retrying up to 5 times. +# +# Making sure the "FROM" image is present locally before building an image +# based on it removes the risk of a "docker build" failing due to transient +# Docker registry problems. +# +# The argument is the path to the Dockerfile to be built. +function prefetch_docker_image { + DOCKERFILE="$1" + IMAGE=$(grep '^FROM' "$DOCKERFILE" | awk '{ print $2 }' | head -1) + ATTEMPT=0 + MAX_RETRIES=5 + + while true + do + ATTEMPT=$((ATTEMPT+1)) + + if [ $ATTEMPT -gt $MAX_RETRIES ] ; then + echo "Docker pull retries exceeded, aborting." + exit 1 + fi + + if docker pull "$IMAGE" ; then + echo "Docker pull of $IMAGE successful." + break + else + echo "Docker pull of $IMAGE unsuccessful, attempt '$ATTEMPT'." + fi + done +} + diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index c610a1dd86..1b1c2b625e 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.." # necessary network access 3rd_party/pull-eigen.sh +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" + for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u` do @@ -72,6 +74,7 @@ do DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_builder/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ + prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" . # Using tar to copy the build artifacts out of the container seems more reliable # than docker cp, and also means the files end up with the correct uid/gid diff --git a/dev-tools/docker_check_style.sh b/dev-tools/docker_check_style.sh index 0ee2a8f7a7..f3f2419ca2 100755 --- a/dev-tools/docker_check_style.sh +++ b/dev-tools/docker_check_style.sh @@ -26,6 +26,8 @@ cd "$TOOLS_DIR/.." DOCKERFILE="$TOOLS_DIR/docker/style_checker/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-style-$$ +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" +prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG -f "$DOCKERFILE" . docker run --rm --workdir=/ml-cpp $TEMP_TAG dev-tools/check-style.sh --all RC=$? diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index f4df21c709..868258e5fd 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -62,6 +62,8 @@ cd "$TOOLS_DIR/.." # necessary network access 3rd_party/pull-eigen.sh +. "$TOOLS_DIR/docker/prefetch_docker_image.sh" + for PLATFORM in `echo $PLATFORMS | tr ' ' '\n' | sort -u` do @@ -73,6 +75,7 @@ do DOCKERFILE="$TOOLS_DIR/docker/${PLATFORM}_tester/Dockerfile" TEMP_TAG=`git rev-parse --short=14 HEAD`-$PLATFORM-$$ + prefetch_docker_image "$DOCKERFILE" docker build --no-cache --force-rm -t $TEMP_TAG --build-arg VERSION_QUALIFIER="$VERSION_QUALIFIER" --build-arg SNAPSHOT=$SNAPSHOT -f "$DOCKERFILE" . # Using tar to copy the build and test artifacts out of the container seems # more reliable than docker cp, and also means the files end up with the diff --git a/dev-tools/jenkins_ci.sh b/dev-tools/jenkins_ci.sh index 3b280157be..9da20ace1f 100755 --- a/dev-tools/jenkins_ci.sh +++ b/dev-tools/jenkins_ci.sh @@ -67,6 +67,9 @@ git repack -a -d readonly GIT_TOPLEVEL=$(git rev-parse --show-toplevel 2> /dev/null) rm -f "${GIT_TOPLEVEL}/.git/objects/info/alternates" +# The Docker version is helpful to identify version-specific Docker bugs +docker --version + # Build and test the native Linux architecture if [ "$HARDWARE_ARCH" = x86_64 ] ; then