From 4fdfb6ed08d94bb0c7a37b014d77d6873687ef01 Mon Sep 17 00:00:00 2001 From: to-bar <46519524+to-bar@users.noreply.github.com> Date: Fri, 12 Mar 2021 20:00:19 +0100 Subject: [PATCH] Fix missing erlang-23.1.4 package for RHEL/CentOS (#2123) * Switch erlang and rabbitmq repo to bintray * HiPE is not supported since Erlang 22 * Update Erlang to 23.1.5 * Update changelog * Add clean metadata for upgrades * Retry download_packages --- CHANGELOG-0.6.md | 1 + .../tasks/install-packages-debian.yml | 17 ++--- .../centos-7/download-requirements.sh | 66 +++++++++++++++---- .../centos-7/requirements.txt | 2 +- .../redhat-7/download-requirements.sh | 66 +++++++++++++++---- .../redhat-7/requirements.txt | 2 +- .../ubuntu-18.04/requirements.txt | 41 ++++++------ .../roles/upgrade/tasks/rabbitmq.yml | 14 ++-- 8 files changed, 141 insertions(+), 68 deletions(-) diff --git a/CHANGELOG-0.6.md b/CHANGELOG-0.6.md index d6fc209275..b6867229ab 100644 --- a/CHANGELOG-0.6.md +++ b/CHANGELOG-0.6.md @@ -6,6 +6,7 @@ - [#1302](https://github.com/epiphany-platform/epiphany/issues/1302) - Ability to update control plane certificates expiration date - [#2115](https://github.com/epiphany-platform/epiphany/issues/2115) - Epicli hangs on importing GPG keys for kubernetes repository on RHEL +- [#2121](https://github.com/epiphany-platform/epiphany/issues/2121) - [RedHat/CentOS] Erlang package versions specified in requirements are missing in external repository ### Fixed diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml index a7552bd535..bd53f5e313 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml @@ -2,28 +2,25 @@ - name: Install packages apt: name: - - logrotate - - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }} - erlang-xmerl={{ versions.debian.erlang }} + - logrotate - rabbitmq-server={{ versions.debian.rabbitmq }} update_cache: true state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh index fff136a2ad..8873c87b4e 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -137,8 +137,7 @@ download_packages() { if [[ -n $packages ]]; then # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages - yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || - exit_with_error "yumdownloader failed for: $packages" + run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3 fi } @@ -351,6 +350,32 @@ remove_installed_packages() { fi } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection) # params: [--no-exit-on-error] run_cmd() { @@ -426,6 +451,15 @@ _get_shell_escaped_array() { fi } +# params: +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # Prints string in format that can be reused as shell input (escapes non-printable characters) _print_array_as_shell_escaped_string() { local output @@ -652,24 +686,25 @@ EOF ) RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' -[rabbitmq_erlang] -name=rabbitmq_erlang -baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch -repo_gpgcheck=1 +[rabbitmq-erlang] +name=rabbitmq-erlang +baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7 gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +deltarpm_percentage=0 EOF ) RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server -baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 +[bintray-rabbitmq-server] +name=bintray-rabbitmq-rpm +baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/ gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey EOF ) @@ -688,8 +723,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' # -> Software Collections (SCL) https://wiki.centos.org/AdditionalResources/Repositories/SCL @@ -704,6 +739,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index cb2fc716e8..6ee5f96495 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -33,7 +33,7 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 elasticsearch-oss-7.3.2 # Open Distro for Elasticsearch -erlang-23.1.4 # must be compatible with rabbitmq version +erlang-23.1.5 # must be compatible with rabbitmq version ethtool filebeat-6.8.5 # actually it's filebeat-oss firewalld diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh index 3e6516fc29..632aebe981 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -137,8 +137,7 @@ download_packages() { if [[ -n $packages ]]; then # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages - yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || - exit_with_error "yumdownloader failed for: $packages" + run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3 fi } @@ -368,6 +367,32 @@ remove_installed_packages() { fi } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection) # params: [--no-exit-on-error] run_cmd() { @@ -443,6 +468,15 @@ _get_shell_escaped_array() { fi } +# params: +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # Prints string in format that can be reused as shell input (escapes non-printable characters) _print_array_as_shell_escaped_string() { local output @@ -682,24 +716,25 @@ EOF ) RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' -[rabbitmq_erlang] -name=rabbitmq_erlang -baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch -repo_gpgcheck=1 +[rabbitmq-erlang] +name=rabbitmq-erlang +baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7 gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +deltarpm_percentage=0 EOF ) RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server -baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 +[bintray-rabbitmq-server] +name=bintray-rabbitmq-rpm +baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/ gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey EOF ) @@ -718,8 +753,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' # some packages are from EPEL repo @@ -727,6 +762,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 9bb3d6b679..c26bf8f988 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -33,7 +33,7 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 elasticsearch-oss-7.3.2 # Open Distro for Elasticsearch -erlang-23.1.4 # must be compatible with rabbitmq version +erlang-23.1.5 # must be compatible with rabbitmq version ethtool filebeat-6.8.5 # actually it's filebeat-oss firewalld diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index ce954a3328..a469f9bc17 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -16,26 +16,27 @@ elasticsearch-curator 5.8.3 elasticsearch-oss 6.8.5 elasticsearch-oss 7.3.2 -# erlang packages must be compatible with rabbitmq version -erlang-eldap 1:23.1.4 -erlang-inets 1:23.1.4 -erlang-os-mon 1:23.1.4 -erlang-public-key 1:23.1.4 -erlang-ssl 1:23.1.4 - -# additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 -erlang-asn1 1:23.1.4 -erlang-base-hipe 1:23.1.4 -erlang-crypto 1:23.1.4 -erlang-ftp 1:23.1.4 -erlang-mnesia 1:23.1.4 -erlang-parsetools 1:23.1.4 -erlang-runtime-tools 1:23.1.4 -erlang-snmp 1:23.1.4 -erlang-syntax-tools 1:23.1.4 -erlang-tftp 1:23.1.4 -erlang-tools 1:23.1.4 -erlang-xmerl 1:23.1.4 +# Erlang packages must be compatible with RabbitMQ version. +# Metapackages such as erlang and erlang-nox must only be used +# with apt version pinning. They do not pin their dependency versions. +# List based on: https://www.rabbitmq.com/install-debian.html#installing-erlang-package +erlang-asn1 1:23.1.5 +erlang-base 1:23.1.5 +erlang-crypto 1:23.1.5 +erlang-eldap 1:23.1.5 +erlang-ftp 1:23.1.5 +erlang-inets 1:23.1.5 +erlang-mnesia 1:23.1.5 +erlang-os-mon 1:23.1.5 +erlang-parsetools 1:23.1.5 +erlang-public-key 1:23.1.5 +erlang-runtime-tools 1:23.1.5 +erlang-snmp 1:23.1.5 +erlang-ssl 1:23.1.5 +erlang-syntax-tools 1:23.1.5 +erlang-tftp 1:23.1.5 +erlang-tools 1:23.1.5 +erlang-xmerl 1:23.1.5 ethtool filebeat 6.8.5 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml index 667c9e1599..5d6e348a16 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml @@ -75,21 +75,19 @@ vars: _packages: Debian: - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix issue #1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }}