diff --git a/CHANGELOG-0.8.md b/CHANGELOG-0.8.md index 0db718f92c..50c9fc8280 100644 --- a/CHANGELOG-0.8.md +++ b/CHANGELOG-0.8.md @@ -5,6 +5,7 @@ ### Fixed - [#2049](https://github.com/epiphany-platform/epiphany/issues/2049) - Elasticsearch-curator installation fails on RedHat +- [#2165](https://github.com/epiphany-platform/epiphany/issues/2165) - Replace Bintray repository ## [0.8.1] 2021-02-10 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/defaults/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/defaults/main.yml index f6eb08b80a..7990e336af 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/defaults/main.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/defaults/main.yml @@ -8,5 +8,5 @@ versions: erlang: 1:23.1.* rabbitmq: 3.8.9* redhat: - erlang: 23.1.* + erlang_filename: erlang-23.1.5-1.el7.x86_64.rpm rabbitmq: 3.8.9 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/meta/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/meta/main.yml new file mode 100644 index 0000000000..745ba4d956 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: preflight_facts diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml index a7552bd535..bd53f5e313 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml @@ -2,28 +2,25 @@ - name: Install packages apt: name: - - logrotate - - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }} - erlang-xmerl={{ versions.debian.erlang }} + - logrotate - rabbitmq-server={{ versions.debian.rabbitmq }} update_cache: true state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml index 1d9d7c77a1..c808205bbe 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml @@ -3,7 +3,7 @@ yum: name: - logrotate - - erlang-{{ versions.redhat.erlang }} + - "{{ repository_url }}/files/{{ versions.redhat.erlang_filename }}" - rabbitmq-server-{{ versions.redhat.rabbitmq }} update_cache: true state: present diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh index f7dcd32e0f..3a5b984539 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh @@ -335,6 +335,33 @@ remove_installed_packages() { fi } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + + # params: run_cmd() { local cmd_arr=("$@") @@ -349,6 +376,16 @@ usage() { [ -z "$1" ] || exit "$1" } +# === Helper function=== + +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # === Start === [ $# -gt 0 ] || usage 1 >&2 @@ -559,25 +596,15 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG EOF ) -RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF' -[rabbitmq_erlang] -name=rabbitmq_erlang -baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch -repo_gpgcheck=1 -gpgcheck=1 -enabled=1 -gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey -EOF -) - RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server +[rabbitmq-server] +name=rabbitmq-server baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 gpgcheck=1 -enabled=1 gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey +repo_gpgcheck=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +enabled=1 EOF ) @@ -594,8 +621,7 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' disable_repo '2ndquadrant-dl-default-release-pg10-debug' @@ -611,6 +637,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index acf2b2b8f6..b0f6f95799 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -34,7 +34,6 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 # for elasticsearch role elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles -erlang-23.1.4 # must be compatible with rabbitmq version ethtool filebeat-7.8.1 firewalld @@ -147,6 +146,8 @@ kubernetes-cni-0.7.5-0 kubernetes-cni-0.8.6-0 [files] +# Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger +https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh index 00f0907b84..8e5891e07a 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh @@ -360,12 +360,48 @@ run_cmd() { "${cmd_arr[@]}" || exit_with_error "Command failed: ${cmd_arr[*]}" } +remove_yum_cache_for_untracked_repos() { + local basearch releasever + basearch=$(uname --machine) + releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3) + local cachedir find_output + cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf) + cachedir="${cachedir/\$basearch/$basearch}" + cachedir="${cachedir/\$releasever/$releasever}" + find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';') + local -a repos_with_cache=() + if [ -n "$find_output" ]; then + readarray -t repos_with_cache <<< "$find_output" + fi + local all_repos_output + all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//') + local -a all_repos=() + readarray -t all_repos <<< "$all_repos_output" + if (( ${#repos_with_cache[@]} > 0 )); then + for cached_repo in "${repos_with_cache[@]}"; do + if ! _in_array "$cached_repo" "${all_repos[@]}"; then + run_cmd rm -rf "$cachedir/$cached_repo" + fi + done + fi +} + usage() { echo "usage: ./$(basename $0) " echo " ./$(basename $0) /tmp/downloads" [ -z "$1" ] || exit "$1" } +# === Helper function=== + +_in_array() { + local value=${1} + shift + local array=( "$@" ) + + (( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value" +} + # === Start === [ $# -gt 0 ] || usage 1 >&2 @@ -601,13 +637,14 @@ EOF ) RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF' -[rabbitmq_rabbitmq-server] -name=rabbitmq_rabbitmq-server +[rabbitmq-server] +name=rabbitmq-server baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch -repo_gpgcheck=1 gpgcheck=1 -enabled=1 gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey +repo_gpgcheck=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +enabled=1 EOF ) @@ -624,8 +661,7 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF" -add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" +add_repo_as_file 'rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF" add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm' disable_repo '2ndquadrant-dl-default-release-pg10-debug' @@ -634,6 +670,9 @@ if ! is_package_installed 'epel-release'; then install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release' fi +# clean metadata for upgrades (when the same package can be downloaded from changed repo) +run_cmd remove_yum_cache_for_untracked_repos + run_cmd yum -y makecache fast # --- Download packages --- diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 4d62ba7f7c..f4ad72f38c 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -33,7 +33,6 @@ ebtables elasticsearch-curator-5.8.3 elasticsearch-oss-6.8.5 # for elasticsearch role elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles -erlang-23.1.4 # must be compatible with rabbitmq version ethtool filebeat-7.8.1 firewalld @@ -144,6 +143,8 @@ kubernetes-cni-0.7.5-0 kubernetes-cni-0.8.6-0 [files] +# Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger +https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh index 8c3f1dae90..bc998003f1 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh @@ -12,9 +12,11 @@ echo "deb https://packages.grafana.com/oss/deb stable main" | tee /etc/apt/sourc wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list -wget -qO - https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc | apt-key add - -echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang-23.x" | tee /etc/apt/sources.list.d/erlang-23.x.list -echo "deb https://dl.bintray.com/rabbitmq/debian bionic main" | tee /etc/apt/sources.list.d/rabbitmq.list +wget -qO - https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc | apt-key add - +echo "deb https://packages.erlang-solutions.com/ubuntu bionic contrib" | tee /etc/apt/sources.list.d/erlang-23.x.list + +wget -qO - https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey | apt-key add - +echo "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu bionic main" | tee /etc/apt/sources.list.d/rabbitmq.list wget -qO - https://download.docker.com/linux/ubuntu/gpg | apt-key add - echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" | tee /etc/apt/sources.list.d/docker-ce.list diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index 33f69882ba..eea306c337 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -16,26 +16,27 @@ elasticsearch-curator 5.8.3 elasticsearch-oss 6.8.5 # for elasticsearch role elasticsearch-oss 7.9.1 # for opendistroforelasticsearch & logging roles -# erlang packages must be compatible with rabbitmq version -erlang-eldap 1:23.1.4 -erlang-inets 1:23.1.4 -erlang-os-mon 1:23.1.4 -erlang-public-key 1:23.1.4 -erlang-ssl 1:23.1.4 - -# additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920 -erlang-asn1 1:23.1.4 -erlang-base-hipe 1:23.1.4 -erlang-crypto 1:23.1.4 -erlang-ftp 1:23.1.4 -erlang-mnesia 1:23.1.4 -erlang-parsetools 1:23.1.4 -erlang-runtime-tools 1:23.1.4 -erlang-snmp 1:23.1.4 -erlang-syntax-tools 1:23.1.4 -erlang-tftp 1:23.1.4 -erlang-tools 1:23.1.4 -erlang-xmerl 1:23.1.4 +# Erlang packages must be compatible with RabbitMQ version. +# Metapackages such as erlang and erlang-nox must only be used +# with apt version pinning. They do not pin their dependency versions. +# List based on: https://www.rabbitmq.com/install-debian.html#installing-erlang-package +erlang-asn1 1:23.1.5 +erlang-base 1:23.1.5 +erlang-crypto 1:23.1.5 +erlang-eldap 1:23.1.5 +erlang-ftp 1:23.1.5 +erlang-inets 1:23.1.5 +erlang-mnesia 1:23.1.5 +erlang-os-mon 1:23.1.5 +erlang-parsetools 1:23.1.5 +erlang-public-key 1:23.1.5 +erlang-runtime-tools 1:23.1.5 +erlang-snmp 1:23.1.5 +erlang-ssl 1:23.1.5 +erlang-syntax-tools 1:23.1.5 +erlang-tftp 1:23.1.5 +erlang-tools 1:23.1.5 +erlang-xmerl 1:23.1.5 ethtool filebeat 7.8.1 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml index 667c9e1599..e809230aa8 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml +++ b/core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml @@ -75,21 +75,19 @@ vars: _packages: Debian: - - erlang-eldap={{ versions.debian.erlang }} - - erlang-inets={{ versions.debian.erlang }} - - erlang-os-mon={{ versions.debian.erlang }} - - erlang-public-key={{ versions.debian.erlang }} - - erlang-ssl={{ versions.debian.erlang }} - - # Additional dependencies required to fix issue #1920 - erlang-asn1={{ versions.debian.erlang }} - - erlang-base-hipe={{ versions.debian.erlang }} + - erlang-base={{ versions.debian.erlang }} - erlang-crypto={{ versions.debian.erlang }} + - erlang-eldap={{ versions.debian.erlang }} - erlang-ftp={{ versions.debian.erlang }} + - erlang-inets={{ versions.debian.erlang }} - erlang-mnesia={{ versions.debian.erlang }} + - erlang-os-mon={{ versions.debian.erlang }} - erlang-parsetools={{ versions.debian.erlang }} + - erlang-public-key={{ versions.debian.erlang }} - erlang-runtime-tools={{ versions.debian.erlang }} - erlang-snmp={{ versions.debian.erlang }} + - erlang-ssl={{ versions.debian.erlang }} - erlang-syntax-tools={{ versions.debian.erlang }} - erlang-tftp={{ versions.debian.erlang }} - erlang-tools={{ versions.debian.erlang }} @@ -97,7 +95,7 @@ - rabbitmq-server={{ versions.debian.rabbitmq }} RedHat: - - erlang-{{ versions.redhat.erlang }} + - "{{ repository_url }}/files/{{ versions.redhat.erlang_filename }}" - rabbitmq-server-{{ versions.redhat.rabbitmq }} - name: RabbitMQ | Start master