From 90b55f8d9dc44586217fc5035b855e9b89bfa02e Mon Sep 17 00:00:00 2001
From: to-bar <46519524+to-bar@users.noreply.github.com>
Date: Thu, 11 Mar 2021 17:06:00 +0100
Subject: [PATCH] Import rpm repo GPG keys with retries (#2120)

* Add 'import_repo_gpg_keys' function with retries

* Fix gpgkey for PostgreSQL repo

* Prefer Docker CE patched repo

* Update changelog
---
 CHANGELOG-0.10.md                             |   1 +
 .../centos-7/download-requirements.sh         | 122 ++++++++++++++++--
 .../redhat-7/download-requirements.sh         | 122 ++++++++++++++++--
 3 files changed, 217 insertions(+), 28 deletions(-)

diff --git a/CHANGELOG-0.10.md b/CHANGELOG-0.10.md
index 10b0342b16..c9e0c66312 100644
--- a/CHANGELOG-0.10.md
+++ b/CHANGELOG-0.10.md
@@ -21,6 +21,7 @@
 - [#2069](https://github.com/epiphany-platform/epiphany/issues/2069) - [CentOS] epicli fails on task [repository : Create epirepo repository]
 - [#2066](https://github.com/epiphany-platform/epiphany/issues/2066) - [CentOS] download-requirements.sh fails on extracting tar with backed up repos
 - [#2067](https://github.com/epiphany-platform/epiphany/issues/2067) - [CentOS] epicli fails on task "repository : Wait for yum lock to be released" on CentOS Minimal
+- [#2115](https://github.com/epiphany-platform/epiphany/issues/2115) - Epicli hangs on importing GPG keys for kubernetes repository on RHEL
 
 ### Updated
 
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
index 0b0d5e9404..c72c07c299 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.0.4
+# VERSION 1.0.5
 
 # NOTE: You can run only one instance of this script, new instance kills the previous one
 #       This limitation is for Ansible
@@ -34,7 +34,13 @@ add_repo_as_file() {
 		echol "Adding repository: $repo_id"
 		cat <<< "$config_file_content" > "/etc/yum.repos.d/$config_file_name" ||
 			exit_with_error "Function add_repo_as_file failed for repo: $repo_id"
-		# to accept import of GPG keys
+		local -a gpg_key_urls
+		IFS=" " read -r -a gpg_key_urls \
+			<<< "$(grep -i --only-matching --perl-regexp '(?<=^gpgkey=)http[^#\n]+' <<< "$config_file_content")"
+		if (( ${#gpg_key_urls[@]} > 0 )); then
+			import_repo_gpg_keys "${gpg_key_urls[@]}" 3
+		fi
+		# to accept import of repo's GPG key (for repo_gpgcheck=1)
 		yum -y repolist > /dev/null || exit_with_error "Command failed: yum -y repolist"
 	fi
 }
@@ -239,6 +245,16 @@ get_unique_array() {
 	eval $result_var_name='("${array[@]}")'
 }
 
+# params: <url(s)> <retries>
+import_repo_gpg_keys() {
+	local retries=${!#} # get last arg
+	local urls=( "${@:1:$# - 1}" )  # remove last arg
+
+	for url in "${urls[@]}"; do
+		run_cmd_with_retries rpm --import "$url" "$retries"
+	done
+}
+
 # params: <package_name_or_url> [package_name]
 install_package() {
 	local package_name_or_url="$1"
@@ -335,12 +351,56 @@ remove_installed_packages() {
 	fi
 }
 
-# params: <command to execute>
+# Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection)
+# params: <command to execute> [--no-exit-on-error]
 run_cmd() {
 	local cmd_arr=("$@")
 
-	echol "Executing: ${cmd_arr[*]}"
-	"${cmd_arr[@]}" || exit_with_error "Command failed: ${cmd_arr[*]}"
+	local exit_on_error=1
+	if [[ ${cmd_arr[-1]} == '--no-exit-on-error' ]]; then
+		exit_on_error=0
+		cmd_arr=( "${cmd_arr[@]:0:$# - 1}" )  # remove last item
+	fi
+
+	local escaped_string return_code
+	escaped_string=$(_print_array_as_shell_escaped_string "${cmd_arr[@]}")
+	echol "Executing: ${escaped_string}"
+	"${cmd_arr[@]}"; return_code=$?
+	if (( return_code != 0 )) && (( exit_on_error )); then
+		exit_with_error "Command failed: ${escaped_string}"
+	else
+		return $return_code
+	fi
+}
+
+# Runs command with retries, doesn't support commands with shell operators (such as pipe or redirection)
+# params: <command to execute> <retries>
+run_cmd_with_retries() {
+	# pop 'retries' argument
+	local retries=${!#}  # get last arg (indirect expansion)
+	set -- "${@:1:$#-1}"  # set new "$@"
+
+	local cmd_arr=("$@")
+	( # sub-shell is used to limit scope for 'set +e'
+		set +e
+		trap - ERR  # disable global trap locally
+		for ((i=0; i <= retries; i++)); do
+			run_cmd "${cmd_arr[@]}" '--no-exit-on-error'
+			return_code=$?
+			if (( return_code == 0 )); then
+				break
+			elif (( i < retries )); then
+				sleep 1
+				echol "retrying ($(( i+1 ))/${retries})"
+			else
+				echol "ERROR: all attempts failed"
+				local escaped_string
+				escaped_string=$(_print_array_as_shell_escaped_string "${cmd_arr[@]}")
+				exit_with_error "Command failed: ${escaped_string}"
+			fi
+		done
+		return $return_code
+	)
 }
 
 usage() {
@@ -349,8 +409,40 @@ usage() {
 	[ -z "$1" ] || exit "$1"
 }
 
+validate_bash_version() {
+	local major_version=${BASH_VERSINFO[0]}
+	local minor_version=${BASH_VERSINFO[1]}
+	local required_version=(4 2)  # (minor major)
+	if (( major_version < ${required_version[0]} )) || (( minor_version < ${required_version[1]} )); then
+		exit_with_error "This script requires Bash version ${required_version[0]}.${required_version[1]} or higher."
+	fi
+}
+
+# === Helper functions (in alphabetical order) ===
+
+_get_shell_escaped_array() {
+	if (( $# > 0 )); then
+		printf '%q\n' "$@"
+	fi
+}
+
+# Prints string in format that can be reused as shell input (escapes non-printable characters)
+_print_array_as_shell_escaped_string() {
+	local output
+	output=$(_get_shell_escaped_array "$@")
+	local escaped=()
+	if [ -n "$output" ]; then
+		readarray -t escaped <<< "$output"
+	fi
+	if (( ${#escaped[@]} > 0 )); then
+		printf '%s\n' "${escaped[*]}"
+	fi
+}
+
 # === Start ===
 
+validate_bash_version
+
 [ $# -gt 0 ] || usage 1 >&2
 readonly START_TIME=$(date +%s)
 
@@ -482,9 +574,9 @@ enable_repo 'extras'
 
 # --- Add repos ---
 
-DOCKER_CE_FALLBACK_REPO_CONF=$(cat <<'EOF'
-[docker-ce-stable-fallback]
-name=Docker CE Stable - fallback centos/7/x86_64/stable
+DOCKER_CE_PATCHED_REPO_CONF=$(cat <<'EOF'
+[docker-ce-stable-patched]
+name=Docker CE Stable - patched centos/7/x86_64/stable
 baseurl=https://download.docker.com/linux/centos/7/x86_64/stable
 enabled=1
 gpgcheck=1
@@ -569,7 +661,7 @@ name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch
 baseurl=https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-$releasever-$basearch
 enabled=1
 gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG
+gpgkey=https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG
 EOF
 )
 
@@ -595,11 +687,13 @@ gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
 EOF
 )
 
-add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo'
-# occasionally docker-ce repo (at https://download.docker.com/linux/centos/7Server/x86_64/stable) is unavailable
-if ! is_repo_available "docker-ce-stable"; then
-	disable_repo "docker-ce-stable"
-	add_repo_as_file 'docker-ce-stable-fallback' "$DOCKER_CE_FALLBACK_REPO_CONF"
+# Official Docker CE repository, added with https://download.docker.com/linux/centos/docker-ce.repo,
+# has broken URL (https://download.docker.com/linux/centos/7Server/x86_64/stable) for longer time.
+# So direct (patched) link is used first if available.
+add_repo_as_file 'docker-ce-stable-patched' "$DOCKER_CE_PATCHED_REPO_CONF"
+if ! is_repo_available "docker-ce-stable-patched"; then
+	disable_repo "docker-ce-stable-patched"
+	add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo'
 fi
 add_repo_as_file 'elastic-6' "$ELASTIC_6_REPO_CONF"
 add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF"
diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
index ed525b87c3..cf9c540785 100644
--- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
+++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# VERSION 1.0.4
+# VERSION 1.0.5
 
 # NOTE: You can run only one instance of this script, new instance kills the previous one
 #       This limitation is for Ansible
@@ -34,7 +34,13 @@ add_repo_as_file() {
 		echol "Adding repository: $repo_id"
 		cat <<< "$config_file_content" > "/etc/yum.repos.d/$config_file_name" ||
 			exit_with_error "Function add_repo_as_file failed for repo: $repo_id"
-		# to accept import of GPG keys
+		local -a gpg_key_urls
+		IFS=" " read -r -a gpg_key_urls \
+			<<< "$(grep -i --only-matching --perl-regexp '(?<=^gpgkey=)http[^#\n]+' <<< "$config_file_content")"
+		if (( ${#gpg_key_urls[@]} > 0 )); then
+			import_repo_gpg_keys "${gpg_key_urls[@]}" 3
+		fi
+		# to accept import of repo's GPG key (for repo_gpgcheck=1)
 		yum -y repolist > /dev/null || exit_with_error "Command failed: yum -y repolist"
 	fi
 }
@@ -256,6 +262,16 @@ get_unique_array() {
 	eval $result_var_name='("${array[@]}")'
 }
 
+# params: <url(s)> <retries>
+import_repo_gpg_keys() {
+	local retries=${!#} # get last arg
+	local urls=( "${@:1:$# - 1}" )  # remove last arg
+
+	for url in "${urls[@]}"; do
+		run_cmd_with_retries rpm --import "$url" "$retries"
+	done
+}
+
 # params: <package_name_or_url> [package_name]
 install_package() {
 	local package_name_or_url="$1"
@@ -352,12 +368,56 @@ remove_installed_packages() {
 	fi
 }
 
-# params: <command to execute>
+# Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection)
+# params: <command to execute> [--no-exit-on-error]
 run_cmd() {
 	local cmd_arr=("$@")
 
-	echol "Executing: ${cmd_arr[*]}"
-	"${cmd_arr[@]}" || exit_with_error "Command failed: ${cmd_arr[*]}"
+	local exit_on_error=1
+	if [[ ${cmd_arr[-1]} == '--no-exit-on-error' ]]; then
+		exit_on_error=0
+		cmd_arr=( "${cmd_arr[@]:0:$# - 1}" )  # remove last item
+	fi
+
+	local escaped_string return_code
+	escaped_string=$(_print_array_as_shell_escaped_string "${cmd_arr[@]}")
+	echol "Executing: ${escaped_string}"
+	"${cmd_arr[@]}"; return_code=$?
+	if (( return_code != 0 )) && (( exit_on_error )); then
+		exit_with_error "Command failed: ${escaped_string}"
+	else
+		return $return_code
+	fi
+}
+
+# Runs command with retries, doesn't support commands with shell operators (such as pipe or redirection)
+# params: <command to execute> <retries>
+run_cmd_with_retries() {
+	# pop 'retries' argument
+	local retries=${!#}  # get last arg (indirect expansion)
+	set -- "${@:1:$#-1}"  # set new "$@"
+
+	local cmd_arr=("$@")
+	( # sub-shell is used to limit scope for 'set +e'
+		set +e
+		trap - ERR  # disable global trap locally
+		for ((i=0; i <= retries; i++)); do
+			run_cmd "${cmd_arr[@]}" '--no-exit-on-error'
+			return_code=$?
+			if (( return_code == 0 )); then
+				break
+			elif (( i < retries )); then
+				sleep 1
+				echol "retrying ($(( i+1 ))/${retries})"
+			else
+				echol "ERROR: all attempts failed"
+				local escaped_string
+				escaped_string=$(_print_array_as_shell_escaped_string "${cmd_arr[@]}")
+				exit_with_error "Command failed: ${escaped_string}"
+			fi
+		done
+		return $return_code
+	)
 }
 
 usage() {
@@ -366,8 +426,40 @@ usage() {
 	[ -z "$1" ] || exit "$1"
 }
 
+validate_bash_version() {
+	local major_version=${BASH_VERSINFO[0]}
+	local minor_version=${BASH_VERSINFO[1]}
+	local required_version=(4 2)  # (minor major)
+	if (( major_version < ${required_version[0]} )) || (( minor_version < ${required_version[1]} )); then
+		exit_with_error "This script requires Bash version ${required_version[0]}.${required_version[1]} or higher."
+	fi
+}
+
+# === Helper functions (in alphabetical order) ===
+
+_get_shell_escaped_array() {
+	if (( $# > 0 )); then
+		printf '%q\n' "$@"
+	fi
+}
+
+# Prints string in format that can be reused as shell input (escapes non-printable characters)
+_print_array_as_shell_escaped_string() {
+	local output
+	output=$(_get_shell_escaped_array "$@")
+	local escaped=()
+	if [ -n "$output" ]; then
+		readarray -t escaped <<< "$output"
+	fi
+	if (( ${#escaped[@]} > 0 )); then
+		printf '%s\n' "${escaped[*]}"
+	fi
+}
+
 # === Start ===
 
+validate_bash_version
+
 [ $# -gt 0 ] || usage 1 >&2
 readonly START_TIME=$(date +%s)
 
@@ -512,9 +604,9 @@ enable_repo "$REPO_ID"
 
 # --- Add repos ---
 
-DOCKER_CE_FALLBACK_REPO_CONF=$(cat <<'EOF'
-[docker-ce-stable-fallback]
-name=Docker CE Stable - fallback centos/7/x86_64/stable
+DOCKER_CE_PATCHED_REPO_CONF=$(cat <<'EOF'
+[docker-ce-stable-patched]
+name=Docker CE Stable - patched centos/7/x86_64/stable
 baseurl=https://download.docker.com/linux/centos/7/x86_64/stable
 enabled=1
 gpgcheck=1
@@ -599,7 +691,7 @@ name=PostgreSQL 10 for RHEL/CentOS $releasever - $basearch
 baseurl=https://download.postgresql.org/pub/repos/yum/10/redhat/rhel-$releasever-$basearch
 enabled=1
 gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG
+gpgkey=https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG
 EOF
 )
 
@@ -625,11 +717,13 @@ gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
 EOF
 )
 
-add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo'
-# occasionally docker-ce repo (at https://download.docker.com/linux/centos/7Server/x86_64/stable) is unavailable
-if ! is_repo_available "docker-ce-stable"; then
-	disable_repo "docker-ce-stable"
-	add_repo_as_file 'docker-ce-stable-fallback' "$DOCKER_CE_FALLBACK_REPO_CONF"
+# Official Docker CE repository, added with https://download.docker.com/linux/centos/docker-ce.repo,
+# has broken URL (https://download.docker.com/linux/centos/7Server/x86_64/stable) for longer time.
+# So direct (patched) link is used first if available.
+add_repo_as_file 'docker-ce-stable-patched' "$DOCKER_CE_PATCHED_REPO_CONF"
+if ! is_repo_available "docker-ce-stable-patched"; then
+	disable_repo "docker-ce-stable-patched"
+	add_repo 'docker-ce' 'https://download.docker.com/linux/centos/docker-ce.repo'
 fi
 add_repo_as_file 'elastic-6' "$ELASTIC_6_REPO_CONF"
 add_repo_as_file 'elasticsearch-7' "$ELASTICSEARCH_7_REPO_CONF"