diff --git a/CHANGELOG-0.10.md b/CHANGELOG-0.10.md index c9e0c66312..bfe16d7edc 100644 --- a/CHANGELOG-0.10.md +++ b/CHANGELOG-0.10.md @@ -22,6 +22,7 @@ - [#2066](https://github.com/epiphany-platform/epiphany/issues/2066) - [CentOS] download-requirements.sh fails on extracting tar with backed up repos - [#2067](https://github.com/epiphany-platform/epiphany/issues/2067) - [CentOS] epicli fails on task "repository : Wait for yum lock to be released" on CentOS Minimal - [#2115](https://github.com/epiphany-platform/epiphany/issues/2115) - Epicli hangs on importing GPG keys for kubernetes repository on RHEL +- [#2121](https://github.com/epiphany-platform/epiphany/issues/2121) - [RedHat/CentOS] Erlang package versions specified in requirements are missing in external repository ### Updated diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml index a7552bd535..bd53f5e313 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml @@ -2,28 +2,25 @@ - name: Install packages apt: name: - - logrotate - - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }} - erlang-xmerl={{ versions.debian.erlang }} + - logrotate - rabbitmq-server={{ versions.debian.rabbitmq }} update_cache: true state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh index c72c07c299..c1a385a02c 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -137,8 +137,7 @@ download_packages() { if [[ -n $packages ]]; then # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages - yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || - exit_with_error "yumdownloader failed for: $packages" + run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3 fi } @@ -351,6 +350,32 @@ remove_installed_packages() { fi } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection) # params: [--no-exit-on-error] run_cmd() { @@ -426,6 +451,15 @@ _get_shell_escaped_array() { fi } +# params: +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # Prints string in format that can be reused as shell input (escapes non-printable characters) _print_array_as_shell_escaped_string() { local output @@ -666,24 +700,25 @@ EOF ) RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' -[rabbitmq_erlang] -name=rabbitmq_erlang -baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch -repo_gpgcheck=1 +[rabbitmq-erlang] +name=rabbitmq-erlang +baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7 gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +deltarpm_percentage=0 EOF ) RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server -baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 +[bintray-rabbitmq-server] +name=bintray-rabbitmq-rpm +baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/ gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey EOF ) @@ -702,8 +737,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' disable_repo '2ndquadrant-dl-default-release-pg10-debug' @@ -719,6 +754,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index 4ad3500b52..d06bb4fa37 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -38,7 +38,7 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 # for elasticsearch role elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles -erlang-23.1.4 # must be compatible with rabbitmq version +erlang-23.1.5 # must be compatible with rabbitmq version ethtool filebeat-7.9.2 firewalld diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh index cf9c540785..79542e3476 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -137,8 +137,7 @@ download_packages() { if [[ -n $packages ]]; then # when using --archlist=x86_64 yumdownloader (yum-utils-1.1.31-52) also downloads i686 packages - yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages || - exit_with_error "yumdownloader failed for: $packages" + run_cmd_with_retries yumdownloader --quiet --archlist=x86_64 --exclude='*i686' --destdir="$dest_dir" $packages 3 fi } @@ -368,6 +367,32 @@ remove_installed_packages() { fi } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + # Runs command as array with printing it, doesn't support commands with shell operators (such as pipe or redirection) # params: [--no-exit-on-error] run_cmd() { @@ -443,6 +468,15 @@ _get_shell_escaped_array() { fi } +# params: +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # Prints string in format that can be reused as shell input (escapes non-printable characters) _print_array_as_shell_escaped_string() { local output @@ -696,24 +730,25 @@ EOF ) RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' -[rabbitmq_erlang] -name=rabbitmq_erlang -baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch -repo_gpgcheck=1 +[rabbitmq-erlang] +name=rabbitmq-erlang +baseurl=https://dl.bintray.com/rabbitmq-erlang/rpm/erlang/23/el/7 gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey +deltarpm_percentage=0 EOF ) RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server -baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 +[bintray-rabbitmq-server] +name=bintray-rabbitmq-rpm +baseurl=https://dl.bintray.com/rabbitmq/rpm/rabbitmq-server/v3.8.x/el/7/ gpgcheck=1 +gpgkey=https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc +repo_gpgcheck=1 enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey EOF ) @@ -732,8 +767,8 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-erlang' "$RABBITMQ_ERLANG_REPO_CONF" +add_repo_as_file 'bintray-rabbitmq-rpm' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' disable_repo '2ndquadrant-dl-default-release-pg10-debug' @@ -742,6 +777,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 5b357fba72..a71b157933 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -36,7 +36,7 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 # for elasticsearch role elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles -erlang-23.1.4 # must be compatible with rabbitmq version +erlang-23.1.5 # must be compatible with rabbitmq version ethtool filebeat-7.9.2 firewalld diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index 61119ca1a2..2528958536 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -19,26 +19,27 @@ elasticsearch-curator 5.8.3 elasticsearch-oss 6.8.5 # for elasticsearch role elasticsearch-oss 7.9.1 # for opendistroforelasticsearch & logging roles -# erlang packages must be compatible with rabbitmq version -erlang-eldap 1:23.1.4 -erlang-inets 1:23.1.4 -erlang-os-mon 1:23.1.4 -erlang-public-key 1:23.1.4 -erlang-ssl 1:23.1.4 - -# additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 -erlang-asn1 1:23.1.4 -erlang-base-hipe 1:23.1.4 -erlang-crypto 1:23.1.4 -erlang-ftp 1:23.1.4 -erlang-mnesia 1:23.1.4 -erlang-parsetools 1:23.1.4 -erlang-runtime-tools 1:23.1.4 -erlang-snmp 1:23.1.4 -erlang-syntax-tools 1:23.1.4 -erlang-tftp 1:23.1.4 -erlang-tools 1:23.1.4 -erlang-xmerl 1:23.1.4 +# Erlang packages must be compatible with RabbitMQ version. +# Metapackages such as erlang and erlang-nox must only be used +# with apt version pinning. They do not pin their dependency versions. +# List based on: https://www.rabbitmq.com/install-debian.html#installing-erlang-package +erlang-asn1 1:23.1.5 +erlang-base 1:23.1.5 +erlang-crypto 1:23.1.5 +erlang-eldap 1:23.1.5 +erlang-ftp 1:23.1.5 +erlang-inets 1:23.1.5 +erlang-mnesia 1:23.1.5 +erlang-os-mon 1:23.1.5 +erlang-parsetools 1:23.1.5 +erlang-public-key 1:23.1.5 +erlang-runtime-tools 1:23.1.5 +erlang-snmp 1:23.1.5 +erlang-ssl 1:23.1.5 +erlang-syntax-tools 1:23.1.5 +erlang-tftp 1:23.1.5 +erlang-tools 1:23.1.5 +erlang-xmerl 1:23.1.5 ethtool filebeat 7.9.2 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml index 667c9e1599..5d6e348a16 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml @@ -75,21 +75,19 @@ vars: _packages: Debian: - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix issue #1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }}