Fix missing erlang-23.1.4 package for RHEL/CentOS (#2123)

* Switch erlang and rabbitmq repo to bintray

* HiPE is not supported since Erlang 22

* Update Erlang to 23.1.5

* Update changelog

* Add clean metadata for upgrades

* Retry download_packages

CHANGELOG-0.10.md

@@ -22,6 +22,7 @@
 - [#2066](https://github.com/epiphany-platform/epiphany/issues/2066) - [CentOS] download-requirements.sh fails on extracting tar with backed up repos
 - [#2067](https://github.com/epiphany-platform/epiphany/issues/2067) - [CentOS] epicli fails on task "repository : Wait for yum lock to be released" on CentOS Minimal
 - [#2115](https://github.com/epiphany-platform/epiphany/issues/2115) - Epicli hangs on importing GPG keys for kubernetes repository on RHEL
+- [#2121](https://github.com/epiphany-platform/epiphany/issues/2121) - [RedHat/CentOS] Erlang package versions specified in requirements are missing in external repository
 
 ### Updated
 

core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml

@@ -2,28 +2,25 @@
 - name: Install packages
   apt:
     name:
-      - logrotate
-
-      - erlang-eldap={{ versions.debian.erlang }}
-      - erlang-inets={{ versions.debian.erlang }}
-      - erlang-os-mon={{ versions.debian.erlang }}
-      - erlang-public-key={{ versions.debian.erlang }}
-      - erlang-ssl={{ versions.debian.erlang }}
-
-      # Additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920
       - erlang-asn1={{ versions.debian.erlang }}
-      - erlang-base-hipe={{ versions.debian.erlang }}
+      - erlang-base={{ versions.debian.erlang }}
       - erlang-crypto={{ versions.debian.erlang }}
+      - erlang-eldap={{ versions.debian.erlang }}
       - erlang-ftp={{ versions.debian.erlang }}
+      - erlang-inets={{ versions.debian.erlang }}
       - erlang-mnesia={{ versions.debian.erlang }}
+      - erlang-os-mon={{ versions.debian.erlang }}
       - erlang-parsetools={{ versions.debian.erlang }}
+      - erlang-public-key={{ versions.debian.erlang }}
       - erlang-runtime-tools={{ versions.debian.erlang }}
       - erlang-snmp={{ versions.debian.erlang }}
+      - erlang-ssl={{ versions.debian.erlang }}
       - erlang-syntax-tools={{ versions.debian.erlang }}
       - erlang-tftp={{ versions.debian.erlang }}
       - erlang-tools={{ versions.debian.erlang }}
       - erlang-xmerl={{ versions.debian.erlang }}
 
+      - logrotate
       - rabbitmq-server={{ versions.debian.rabbitmq }}
     update_cache: true
     state: present

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh

@@ -137,8 +137,7 @@ download_packages() {
 
 	if [[ -n $packages ]]; then
 		# when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages
-		yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages ||
-			exit_with_error "yumdownloader failed for: $packages"
+		run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3
 	fi
 }
 
@@ -351,6 +350,32 @@ remove_installed_packages() {
 	fi
 }
 
+remove_yum_cache_for_untracked_repos() {
+	local basearch releasever
+	basearch=$(uname --machine)
+	releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3)
+	local cachedir find_output
+	cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf)
+	cachedir="${cachedir/\$basearch/$basearch}"
+	cachedir="${cachedir/\$releasever/$releasever}"
+	find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';')
+	local -a repos_with_cache=()
+	if [ -n "$find_output" ]; then
+		readarray -t repos_with_cache <<< "$find_output"
+	fi
+	local all_repos_output
+	all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//')
+	local -a all_repos=()
+	readarray -t all_repos <<< "$all_repos_output"
+	if (( ${#repos_with_cache[@]} > 0 )); then
+		for cached_repo in "${repos_with_cache[@]}"; do
+			if ! _in_array "$cached_repo" "${all_repos[@]}"; then
+				run_cmd rm -rf "$cachedir/$cached_repo"
+			fi
+		done
+	fi
+}
+
 # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection)
 # params: <command to execute> [--no-exit-on-error]
 run_cmd() {
@@ -426,6 +451,15 @@ _get_shell_escaped_array() {
 	fi
 }
 
+# params: <value to test> <array>
+_in_array() {
+	local value=${1}
+	shift
+	local array=( "$@" )
+
+	(( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value"
+}
+
 # Prints string in format that can be reused as shell input (escapes non-printable characters)
 _print_array_as_shell_escaped_string() {
 	local output
@@ -666,24 +700,25 @@ EOF
 )
 
 RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_erlang]
-name=rabbitmq_erlang
-baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch
-repo_gpgcheck=1
+[rabbitmq-erlang]
+name=rabbitmq-erlang
+baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7
 gpgcheck=1
+gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
+repo_gpgcheck=1
 enabled=1
-gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey
+deltarpm_percentage=0
 EOF
 )
 
 RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_rabbitmq-server]
-name=rabbitmq_rabbitmq-server
-baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch
-repo_gpgcheck=1
+[bintray-rabbitmq-server]
+name=bintray-rabbitmq-rpm
+baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/
 gpgcheck=1
+gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
+repo_gpgcheck=1
 enabled=1
-gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
 EOF
 )
 
@@ -702,8 +737,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
 add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
-add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF"
-add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
+add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF"
+add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
 disable_repo '2ndquadrant-dl-default-release-pg10-debug'
 
@@ -719,6 +754,9 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
+# clean metadata for upgrades (when the same package can be downloaded from changed repo)
+run_cmd remove_yum_cache_for_untracked_repos
+
 run_cmd yum -y makecache fast
 
 # --- Download packages ---

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -38,7 +38,7 @@ ebtables
 elasticsearch-curator-5.8.3
 elasticsearch-oss-6.8.5 # for elasticsearch role
 elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles
-erlang-23.1.4 # must be compatible with rabbitmq version
+erlang-23.1.5 # must be compatible with rabbitmq version
 ethtool
 filebeat-7.9.2
 firewalld

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh

@@ -137,8 +137,7 @@ download_packages() {
 
 	if [[ -n $packages ]]; then
 		# when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages
-		yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages ||
-			exit_with_error "yumdownloader failed for: $packages"
+		run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3
 	fi
 }
 
@@ -368,6 +367,32 @@ remove_installed_packages() {
 	fi
 }
 
+remove_yum_cache_for_untracked_repos() {
+	local basearch releasever
+	basearch=$(uname --machine)
+	releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3)
+	local cachedir find_output
+	cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf)
+	cachedir="${cachedir/\$basearch/$basearch}"
+	cachedir="${cachedir/\$releasever/$releasever}"
+	find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';')
+	local -a repos_with_cache=()
+	if [ -n "$find_output" ]; then
+		readarray -t repos_with_cache <<< "$find_output"
+	fi
+	local all_repos_output
+	all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//')
+	local -a all_repos=()
+	readarray -t all_repos <<< "$all_repos_output"
+	if (( ${#repos_with_cache[@]} > 0 )); then
+		for cached_repo in "${repos_with_cache[@]}"; do
+			if ! _in_array "$cached_repo" "${all_repos[@]}"; then
+				run_cmd rm -rf "$cachedir/$cached_repo"
+			fi
+		done
+	fi
+}
+
 # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection)
 # params: <command to execute> [--no-exit-on-error]
 run_cmd() {
@@ -443,6 +468,15 @@ _get_shell_escaped_array() {
 	fi
 }
 
+# params: <value to test> <array>
+_in_array() {
+	local value=${1}
+	shift
+	local array=( "$@" )
+
+	(( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value"
+}
+
 # Prints string in format that can be reused as shell input (escapes non-printable characters)
 _print_array_as_shell_escaped_string() {
 	local output
@@ -696,24 +730,25 @@ EOF
 )
 
 RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_erlang]
-name=rabbitmq_erlang
-baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch
-repo_gpgcheck=1
+[rabbitmq-erlang]
+name=rabbitmq-erlang
+baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7
 gpgcheck=1
+gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
+repo_gpgcheck=1
 enabled=1
-gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey
+deltarpm_percentage=0
 EOF
 )
 
 RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_rabbitmq-server]
-name=rabbitmq_rabbitmq-server
-baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch
-repo_gpgcheck=1
+[bintray-rabbitmq-server]
+name=bintray-rabbitmq-rpm
+baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/
 gpgcheck=1
+gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc
+repo_gpgcheck=1
 enabled=1
-gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
 EOF
 )
 
@@ -732,8 +767,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
 add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
-add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF"
-add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
+add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF"
+add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
 disable_repo '2ndquadrant-dl-default-release-pg10-debug'
 
@@ -742,6 +777,9 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
+# clean metadata for upgrades (when the same package can be downloaded from changed repo)
+run_cmd remove_yum_cache_for_untracked_repos
+
 run_cmd yum -y makecache fast
 
 # --- Download packages ---

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -36,7 +36,7 @@ ebtables
 elasticsearch-curator-5.8.3
 elasticsearch-oss-6.8.5 # for elasticsearch role
 elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles
-erlang-23.1.4 # must be compatible with rabbitmq version
+erlang-23.1.5 # must be compatible with rabbitmq version
 ethtool
 filebeat-7.9.2
 firewalld

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt

@@ -19,26 +19,27 @@ elasticsearch-curator 5.8.3
 elasticsearch-oss 6.8.5 # for elasticsearch role
 elasticsearch-oss 7.9.1 # for opendistroforelasticsearch & logging roles
 
-# erlang packages must be compatible with rabbitmq version
-erlang-eldap 1:23.1.4
-erlang-inets 1:23.1.4
-erlang-os-mon 1:23.1.4
-erlang-public-key 1:23.1.4
-erlang-ssl 1:23.1.4
-
-# additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920
-erlang-asn1 1:23.1.4
-erlang-base-hipe 1:23.1.4
-erlang-crypto 1:23.1.4
-erlang-ftp 1:23.1.4
-erlang-mnesia 1:23.1.4
-erlang-parsetools 1:23.1.4
-erlang-runtime-tools 1:23.1.4
-erlang-snmp 1:23.1.4
-erlang-syntax-tools 1:23.1.4
-erlang-tftp 1:23.1.4
-erlang-tools 1:23.1.4
-erlang-xmerl 1:23.1.4
+# Erlang packages must be compatible with RabbitMQ version.
+# Metapackages such as erlang and erlang-nox must only be used
+# with apt version pinning. They do not pin their dependency versions.
+# List based on: https://www.rabbitmq.com/install-debian.html#installing-erlang-package
+erlang-asn1 1:23.1.5
+erlang-base 1:23.1.5
+erlang-crypto 1:23.1.5
+erlang-eldap 1:23.1.5
+erlang-ftp 1:23.1.5
+erlang-inets 1:23.1.5
+erlang-mnesia 1:23.1.5
+erlang-os-mon 1:23.1.5
+erlang-parsetools 1:23.1.5
+erlang-public-key 1:23.1.5
+erlang-runtime-tools 1:23.1.5
+erlang-snmp 1:23.1.5
+erlang-ssl 1:23.1.5
+erlang-syntax-tools 1:23.1.5
+erlang-tftp 1:23.1.5
+erlang-tools 1:23.1.5
+erlang-xmerl 1:23.1.5
 
 ethtool
 filebeat 7.9.2

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml

@@ -75,21 +75,19 @@
       vars:
         _packages:
           Debian:
-            - erlang-eldap={{ versions.debian.erlang }}
-            - erlang-inets={{ versions.debian.erlang }}
-            - erlang-os-mon={{ versions.debian.erlang }}
-            - erlang-public-key={{ versions.debian.erlang }}
-            - erlang-ssl={{ versions.debian.erlang }}
-
-            # Additional dependencies required to fix issue #1920
             - erlang-asn1={{ versions.debian.erlang }}
-            - erlang-base-hipe={{ versions.debian.erlang }}
+            - erlang-base={{ versions.debian.erlang }}
             - erlang-crypto={{ versions.debian.erlang }}
+            - erlang-eldap={{ versions.debian.erlang }}
             - erlang-ftp={{ versions.debian.erlang }}
+            - erlang-inets={{ versions.debian.erlang }}
             - erlang-mnesia={{ versions.debian.erlang }}
+            - erlang-os-mon={{ versions.debian.erlang }}
             - erlang-parsetools={{ versions.debian.erlang }}
+            - erlang-public-key={{ versions.debian.erlang }}
             - erlang-runtime-tools={{ versions.debian.erlang }}
             - erlang-snmp={{ versions.debian.erlang }}
+            - erlang-ssl={{ versions.debian.erlang }}
             - erlang-syntax-tools={{ versions.debian.erlang }}
             - erlang-tftp={{ versions.debian.erlang }}
             - erlang-tools={{ versions.debian.erlang }}