[BACKPORT to v0.8] Replacing Bintray repository

CHANGELOG-0.8.md

@@ -5,6 +5,7 @@
 ### Fixed
 
 - [#2049](https://github.com/epiphany-platform/epiphany/issues/2049) - Elasticsearch-curator installation fails on RedHat
+- [#2165](https://github.com/epiphany-platform/epiphany/issues/2165) - Replace Bintray repository
 
 ## [0.8.1] 2021-02-10
 

core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/defaults/main.yml

@@ -8,5 +8,5 @@ versions:
     erlang: 1:23.1.*
     rabbitmq: 3.8.9*
   redhat:
-    erlang: 23.1.*
+    erlang_filename: erlang-23.1.5-1.el7.x86_64.rpm
     rabbitmq: 3.8.9

core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: preflight_facts

core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-debian.yml

@@ -2,28 +2,25 @@
 - name: Install packages
   apt:
     name:
-      - logrotate
-
-      - erlang-eldap={{ versions.debian.erlang }}
-      - erlang-inets={{ versions.debian.erlang }}
-      - erlang-os-mon={{ versions.debian.erlang }}
-      - erlang-public-key={{ versions.debian.erlang }}
-      - erlang-ssl={{ versions.debian.erlang }}
-
-      # Additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920
       - erlang-asn1={{ versions.debian.erlang }}
-      - erlang-base-hipe={{ versions.debian.erlang }}
+      - erlang-base={{ versions.debian.erlang }}
       - erlang-crypto={{ versions.debian.erlang }}
+      - erlang-eldap={{ versions.debian.erlang }}
       - erlang-ftp={{ versions.debian.erlang }}
+      - erlang-inets={{ versions.debian.erlang }}
       - erlang-mnesia={{ versions.debian.erlang }}
+      - erlang-os-mon={{ versions.debian.erlang }}
       - erlang-parsetools={{ versions.debian.erlang }}
+      - erlang-public-key={{ versions.debian.erlang }}
       - erlang-runtime-tools={{ versions.debian.erlang }}
       - erlang-snmp={{ versions.debian.erlang }}
+      - erlang-ssl={{ versions.debian.erlang }}
       - erlang-syntax-tools={{ versions.debian.erlang }}
       - erlang-tftp={{ versions.debian.erlang }}
       - erlang-tools={{ versions.debian.erlang }}
       - erlang-xmerl={{ versions.debian.erlang }}
 
+      - logrotate
       - rabbitmq-server={{ versions.debian.rabbitmq }}
     update_cache: true
     state: present

core/src/epicli/data/common/ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml

@@ -3,7 +3,7 @@
   yum:
     name:
       - logrotate
-      - erlang-{{ versions.redhat.erlang }}
+      - "{{ repository_url }}/files/{{ versions.redhat.erlang_filename }}"
       - rabbitmq-server-{{ versions.redhat.rabbitmq }}
     update_cache: true
     state: present

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh

@@ -335,6 +335,33 @@ remove_installed_packages() {
 	fi
 }
 
+remove_yum_cache_for_untracked_repos() {
+	local basearch releasever
+	basearch=$(uname --machine)
+	releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3)
+	local cachedir find_output
+	cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf)
+	cachedir="${cachedir/\$basearch/$basearch}"
+	cachedir="${cachedir/\$releasever/$releasever}"
+	find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';')
+	local -a repos_with_cache=()
+	if [ -n "$find_output" ]; then
+		readarray -t repos_with_cache <<< "$find_output"
+	fi
+	local all_repos_output
+	all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//')
+	local -a all_repos=()
+	readarray -t all_repos <<< "$all_repos_output"
+	if (( ${#repos_with_cache[@]} > 0 )); then
+		for cached_repo in "${repos_with_cache[@]}"; do
+			if ! _in_array "$cached_repo" "${all_repos[@]}"; then
+				run_cmd rm -rf "$cachedir/$cached_repo"
+			fi
+		done
+	fi
+}
+
+
 # params: <command to execute>
 run_cmd() {
 	local cmd_arr=("$@")
@@ -349,6 +376,16 @@ usage() {
 	[ -z "$1" ] || exit "$1"
 }
 
+# === Helper function===
+
+_in_array() {
+	local value=${1}
+	shift
+	local array=( "$@" )
+
+	(( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value"
+}
+
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
@@ -559,25 +596,15 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-PGDG
 EOF
 )
 
-RABBITMQ_ERLANG_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_erlang]
-name=rabbitmq_erlang
-baseurl=https://packagecloud.io/rabbitmq/erlang/el/7/$basearch
-repo_gpgcheck=1
-gpgcheck=1
-enabled=1
-gpgkey=https://packagecloud.io/rabbitmq/erlang/gpgkey
-EOF
-)
-
 RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_rabbitmq-server]
-name=rabbitmq_rabbitmq-server
+[rabbitmq-server]
+name=rabbitmq-server
 baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch
-repo_gpgcheck=1
 gpgcheck=1
-enabled=1
 gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
+repo_gpgcheck=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+enabled=1
 EOF
 )
 
@@ -594,8 +621,7 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
 add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
-add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF"
-add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
+add_repo_as_file 'rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
 disable_repo '2ndquadrant-dl-default-release-pg10-debug'
 
@@ -611,6 +637,9 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
+# clean metadata for upgrades (when the same package can be downloaded from changed repo)
+run_cmd remove_yum_cache_for_untracked_repos
+
 run_cmd yum -y makecache fast
 
 # --- Download packages ---

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -34,7 +34,6 @@ ebtables
 elasticsearch-curator-5.8.3
 elasticsearch-oss-6.8.5 # for elasticsearch role
 elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles
-erlang-23.1.4 # must be compatible with rabbitmq version
 ethtool
 filebeat-7.8.1
 firewalld
@@ -147,6 +146,8 @@ kubernetes-cni-0.7.5-0
 kubernetes-cni-0.8.6-0
 
 [files]
+# Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger
+https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm
 https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh

@@ -360,12 +360,48 @@ run_cmd() {
 	"${cmd_arr[@]}" || exit_with_error "Command failed: ${cmd_arr[*]}"
 }
 
+remove_yum_cache_for_untracked_repos() {
+	local basearch releasever
+	basearch=$(uname --machine)
+	releasever=$(rpm -q --provides "$(rpm -q --whatprovides 'system-release(releasever)')" | grep "system-release(releasever)" | cut -d ' ' -f 3)
+	local cachedir find_output
+	cachedir=$(grep --only-matching --perl-regexp '(?<=^cachedir=)[^#\n]+' /etc/yum.conf)
+	cachedir="${cachedir/\$basearch/$basearch}"
+	cachedir="${cachedir/\$releasever/$releasever}"
+	find_output=$(find "$cachedir" -mindepth 1 -maxdepth 1 -type d -exec basename '{}' ';')
+	local -a repos_with_cache=()
+	if [ -n "$find_output" ]; then
+		readarray -t repos_with_cache <<< "$find_output"
+	fi
+	local all_repos_output
+	all_repos_output=$(yum repolist -v all | grep --only-matching --perl-regexp '(?<=^Repo-id)[^/]+' | sed -e 's/^[[:space:]:]*//')
+	local -a all_repos=()
+	readarray -t all_repos <<< "$all_repos_output"
+	if (( ${#repos_with_cache[@]} > 0 )); then
+		for cached_repo in "${repos_with_cache[@]}"; do
+			if ! _in_array "$cached_repo" "${all_repos[@]}"; then
+				run_cmd rm -rf "$cachedir/$cached_repo"
+			fi
+		done
+	fi
+}
+
 usage() {
 	echo "usage: ./$(basename $0) <downloads_dir>"
 	echo "       ./$(basename $0) /tmp/downloads"
 	[ -z "$1" ] || exit "$1"
 }
 
+# === Helper function===
+
+_in_array() {
+	local value=${1}
+	shift
+	local array=( "$@" )
+
+	(( ${#array[@]} > 0 )) && printf '%s\n' "${array[@]}" | grep -q -Fx "$value"
+}
+
 # === Start ===
 
 [ $# -gt 0 ] || usage 1 >&2
@@ -601,13 +637,14 @@ EOF
 )
 
 RABBITMQ_SERVER_REPO_CONF=$(cat <<'EOF'
-[rabbitmq_rabbitmq-server]
-name=rabbitmq_rabbitmq-server
+[rabbitmq-server]
+name=rabbitmq-server
 baseurl=https://packagecloud.io/rabbitmq/rabbitmq-server/el/7/$basearch
-repo_gpgcheck=1
 gpgcheck=1
-enabled=1
 gpgkey=https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey
+repo_gpgcheck=1
+sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+enabled=1
 EOF
 )
 
@@ -624,8 +661,7 @@ add_repo_as_file 'grafana' "$GRAFANA_REPO_CONF"
 add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF"
 add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF"
 add_repo_as_file 'postgresql-10' "$POSTGRESQL_REPO_CONF"
-add_repo_as_file 'rabbitmq_erlang' "$RABBITMQ_ERLANG_REPO_CONF"
-add_repo_as_file 'rabbitmq_rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
+add_repo_as_file 'rabbitmq-server' "$RABBITMQ_SERVER_REPO_CONF"
 add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/10/rpm'
 disable_repo '2ndquadrant-dl-default-release-pg10-debug'
 
@@ -634,6 +670,9 @@ if ! is_package_installed 'epel-release'; then
 	install_package 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' 'epel-release'
 fi
 
+# clean metadata for upgrades (when the same package can be downloaded from changed repo)
+run_cmd remove_yum_cache_for_untracked_repos
+
 run_cmd yum -y makecache fast
 
 # --- Download packages ---

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -33,7 +33,6 @@ ebtables
 elasticsearch-curator-5.8.3
 elasticsearch-oss-6.8.5 # for elasticsearch role
 elasticsearch-oss-7.9.1 # for opendistroforelasticsearch & logging roles
-erlang-23.1.4 # must be compatible with rabbitmq version
 ethtool
 filebeat-7.8.1
 firewalld
@@ -144,6 +143,8 @@ kubernetes-cni-0.7.5-0
 kubernetes-cni-0.8.6-0
 
 [files]
+# Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger
+https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm
 https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.14.0/jmx_prometheus_javaagent-0.14.0.jar
 https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/add-repositories.sh

@@ -12,9 +12,11 @@ echo "deb https://packages.grafana.com/oss/deb stable main" | tee /etc/apt/sourc
 wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
 echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
 
-wget -qO - https://dl.bintray.com/rabbitmq/Keys/rabbitmq-release-signing-key.asc | apt-key add -
-echo "deb http://dl.bintray.com/rabbitmq-erlang/debian bionic erlang-23.x" | tee /etc/apt/sources.list.d/erlang-23.x.list
-echo "deb https://dl.bintray.com/rabbitmq/debian bionic main" | tee /etc/apt/sources.list.d/rabbitmq.list
+wget -qO - https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc | apt-key add -
+echo "deb https://packages.erlang-solutions.com/ubuntu bionic contrib" | tee /etc/apt/sources.list.d/erlang-23.x.list
+
+wget -qO - https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey | apt-key add -
+echo "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu bionic main" | tee /etc/apt/sources.list.d/rabbitmq.list
 
 wget -qO -  https://download.docker.com/linux/ubuntu/gpg | apt-key add -
 echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" | tee /etc/apt/sources.list.d/docker-ce.list

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt

@@ -16,26 +16,27 @@ elasticsearch-curator 5.8.3
 elasticsearch-oss 6.8.5 # for elasticsearch role
 elasticsearch-oss 7.9.1 # for opendistroforelasticsearch & logging roles
 
-# erlang packages must be compatible with rabbitmq version
-erlang-eldap 1:23.1.4
-erlang-inets 1:23.1.4
-erlang-os-mon 1:23.1.4
-erlang-public-key 1:23.1.4
-erlang-ssl 1:23.1.4
-
-# additional dependencies required to fix https://github.com/epiphany-platform/epiphany/issues/1920
-erlang-asn1 1:23.1.4
-erlang-base-hipe 1:23.1.4
-erlang-crypto 1:23.1.4
-erlang-ftp 1:23.1.4
-erlang-mnesia 1:23.1.4
-erlang-parsetools 1:23.1.4
-erlang-runtime-tools 1:23.1.4
-erlang-snmp 1:23.1.4
-erlang-syntax-tools 1:23.1.4
-erlang-tftp 1:23.1.4
-erlang-tools 1:23.1.4
-erlang-xmerl 1:23.1.4
+# Erlang packages must be compatible with RabbitMQ version.
+# Metapackages such as erlang and erlang-nox must only be used
+# with apt version pinning. They do not pin their dependency versions.
+# List based on: https://www.rabbitmq.com/install-debian.html#installing-erlang-package
+erlang-asn1 1:23.1.5
+erlang-base 1:23.1.5
+erlang-crypto 1:23.1.5
+erlang-eldap 1:23.1.5
+erlang-ftp 1:23.1.5
+erlang-inets 1:23.1.5
+erlang-mnesia 1:23.1.5
+erlang-os-mon 1:23.1.5
+erlang-parsetools 1:23.1.5
+erlang-public-key 1:23.1.5
+erlang-runtime-tools 1:23.1.5
+erlang-snmp 1:23.1.5
+erlang-ssl 1:23.1.5
+erlang-syntax-tools 1:23.1.5
+erlang-tftp 1:23.1.5
+erlang-tools 1:23.1.5
+erlang-xmerl 1:23.1.5
 
 ethtool
 filebeat 7.8.1

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/rabbitmq.yml

@@ -75,29 +75,27 @@
       vars:
         _packages:
           Debian:
-            - erlang-eldap={{ versions.debian.erlang }}
-            - erlang-inets={{ versions.debian.erlang }}
-            - erlang-os-mon={{ versions.debian.erlang }}
-            - erlang-public-key={{ versions.debian.erlang }}
-            - erlang-ssl={{ versions.debian.erlang }}
-
-            # Additional dependencies required to fix issue #1920
             - erlang-asn1={{ versions.debian.erlang }}
-            - erlang-base-hipe={{ versions.debian.erlang }}
+            - erlang-base={{ versions.debian.erlang }}
             - erlang-crypto={{ versions.debian.erlang }}
+            - erlang-eldap={{ versions.debian.erlang }}
             - erlang-ftp={{ versions.debian.erlang }}
+            - erlang-inets={{ versions.debian.erlang }}
             - erlang-mnesia={{ versions.debian.erlang }}
+            - erlang-os-mon={{ versions.debian.erlang }}
             - erlang-parsetools={{ versions.debian.erlang }}
+            - erlang-public-key={{ versions.debian.erlang }}
             - erlang-runtime-tools={{ versions.debian.erlang }}
             - erlang-snmp={{ versions.debian.erlang }}
+            - erlang-ssl={{ versions.debian.erlang }}
             - erlang-syntax-tools={{ versions.debian.erlang }}
             - erlang-tftp={{ versions.debian.erlang }}
             - erlang-tools={{ versions.debian.erlang }}
             - erlang-xmerl={{ versions.debian.erlang }}
 
             - rabbitmq-server={{ versions.debian.rabbitmq }}
           RedHat:
-            - erlang-{{ versions.redhat.erlang }}
+            - "{{ repository_url }}/files/{{ versions.redhat.erlang_filename }}"
             - rabbitmq-server-{{ versions.redhat.rabbitmq }}
 
     - name: RabbitMQ | Start master