Download files with deb/rpm extensions to packages subdirectory (#3015)…

… (#3020)

* Download files with deb and rpm extensions to packages subdirectory (#3015)

ansible/playbooks/roles/elasticsearch_curator/defaults/main.yml

@@ -1,7 +1,4 @@
 ---
-curator_package:
-  filename:
-    Debian: elasticsearch-curator_5.8.3_amd64.deb
 curator_version: "5.8.3" # also used by upgrade playbook
 
 elasticsearch_host_ip: "{{ ansible_default_ipv4.address | default(ansible_all_ipv4_addresses[0]) }}"

ansible/playbooks/roles/elasticsearch_curator/tasks/install-es-curator-Debian.yml

@@ -3,5 +3,5 @@
 
 - name: Install elasticsearch-curator package
   apt:
-    deb: "{{ repository_url }}/files/{{ curator_package.filename.Debian }}"
+    name: "elasticsearch-curator={{ curator_version }}"
     state: present

ansible/playbooks/roles/grafana/defaults/versions.yml

@@ -2,10 +2,8 @@
 grafana_version: 8.3.2
 
 grafana_package:
-  filename:
-    Debian:
-      aarch64: null
-      x86_64: grafana_8.3.2_amd64.deb
-    RedHat:
-      aarch64: grafana-8.3.2-1.aarch64.rpm
-      x86_64: grafana-8.3.2-1.x86_64.rpm
+  Debian:
+    x86_64: grafana:amd64=8.3.2
+  RedHat:
+    aarch64: grafana-8.3.2-1.aarch64
+    x86_64: grafana-8.3.2-1.x86_64

ansible/playbooks/roles/grafana/tasks/install-packages-Debian.yml

@@ -1,5 +1,5 @@
 ---
 - name: Install Grafana with dependencies
   apt:
-    deb: "{{ _package_url }}"
+    name: "{{ _package_name }}"
     state: present

ansible/playbooks/roles/grafana/tasks/install-packages-RedHat.yml

@@ -1,7 +1,7 @@
 ---
 - name: Install Grafana with dependencies
   yum:
-    name: "{{ _package_url }}"
+    name: "{{ _package_name }}"
     state: present
   module_defaults:
     yum: { lock_timeout: "{{ yum_lock_timeout }}" }

ansible/playbooks/roles/grafana/tasks/install.yml

@@ -15,7 +15,7 @@
 
 - include_tasks: install-packages-{{ ansible_os_family }}.yml
   vars:
-    _package_url: "{{ repository_url }}/files/{{ grafana_package.filename[ansible_os_family][ansible_architecture] }}"
+    _package_name: "{{ grafana_package[ansible_os_family][ansible_architecture] }}"
 
 - name: Recursively chown /etc/grafana (fix permissions)
   file:

ansible/playbooks/roles/rabbitmq/defaults/main.yml

@@ -8,7 +8,7 @@ versions:
     erlang: 1:23.1.*
     rabbitmq: 3.8.9*
   redhat:
-    erlang_filename: 
-      x86_64: erlang-23.1.5-1.el7.x86_64.rpm
-      aarch64: esl-erlang_23.1.5-1~centos~7_arm64.rpm  
+    erlang_package:
+      x86_64: erlang-23.1.5-*
+      aarch64: esl-erlang-23.1.5-*
     rabbitmq: 3.8.9

ansible/playbooks/roles/rabbitmq/tasks/install-packages-redhat.yml

@@ -3,7 +3,7 @@
   yum:
     name:
       - logrotate
-      - "{{ repository_url }}/files/{{ versions.redhat.erlang_filename[ansible_architecture] }}"
+      - "{{ versions.redhat.erlang_package[ansible_architecture] }}"
       - rabbitmq-server-{{ versions.redhat.rabbitmq }}
     update_cache: true
     state: present

ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat-7/packages.yml

@@ -16,129 +16,138 @@ prereq-packages:
   - 'yum-utils'
 
 packages:
-  - 'audit'  # for docker-ce
-  - 'bash-completion'
-  - 'ca-certificates'
-  - 'cifs-utils'
-  - 'conntrack-tools'  # for kubelet
-  - 'containerd.io-1.4.12'
-  - 'container-selinux'
-  - 'cri-tools-1.13.0'
-  - 'curl'
-  - 'dejavu-sans-fonts'  # for grafana
-  - 'docker-ce-20.10.8'
-  - 'docker-ce-cli-20.10.8'
-  - 'docker-ce-rootless-extras-20.10.8'
-  - 'ebtables'
-  - 'elasticsearch-curator-5.8.3'
-  - 'elasticsearch-oss-7.10.2'  # for opendistroforelasticsearch & logging roles
-  - 'ethtool'
-  - 'filebeat-7.9.2'
-  - 'firewalld'
-  - 'fontconfig'  # for grafana
-  - 'fping'
-  - 'fuse-overlayfs'  # for docker-ce-rootless-extras
-  - 'gnutls'  # for cifs-utils
-  - 'gssproxy'  # for nfs-utils
-  - 'htop'
-  - 'iftop'
-  - 'ipset'  # for firewalld
-  - 'java-1.8.0-openjdk-headless'
-  - 'javapackages-tools'  # for java-1.8.0-openjdk-headless
-  - 'jq'
-  - 'libini_config'  # for nfs-utils
-  - 'libselinux-python'
-  - 'libsemanage-python'
-  - 'libX11'  # for grafana
-  - 'libxcb'  # for grafana
-  - 'libXcursor'  # for grafana
-  - 'libXt'  # for grafana
-  - 'logrotate'
-  - 'net-tools'
-  - 'nfs-utils'
-  - 'nmap-ncat'
+  from_repo:
+    - 'audit'  # for docker-ce
+    - 'bash-completion'
+    - 'ca-certificates'
+    - 'cifs-utils'
+    - 'conntrack-tools'  # for kubelet
+    - 'containerd.io-1.4.12'
+    - 'container-selinux'
+    - 'cri-tools-1.13.0'
+    - 'curl'
+    - 'dejavu-sans-fonts'  # for grafana
+    - 'docker-ce-20.10.8'
+    - 'docker-ce-cli-20.10.8'
+    - 'docker-ce-rootless-extras-20.10.8'
+    - 'ebtables'
+    - 'elasticsearch-curator-5.8.3'
+    - 'elasticsearch-oss-7.10.2'  # for opendistroforelasticsearch & logging roles
+    - 'ethtool'
+    - 'filebeat-7.9.2'
+    - 'firewalld'
+    - 'fontconfig'  # for grafana
+    - 'fping'
+    - 'fuse-overlayfs'  # for docker-ce-rootless-extras
+    - 'gnutls'  # for cifs-utils
+    - 'gssproxy'  # for nfs-utils
+    - 'htop'
+    - 'iftop'
+    - 'ipset'  # for firewalld
+    - 'java-1.8.0-openjdk-headless'
+    - 'javapackages-tools'  # for java-1.8.0-openjdk-headless
+    - 'jq'
+    - 'libini_config'  # for nfs-utils
+    - 'libselinux-python'
+    - 'libsemanage-python'
+    - 'libX11'  # for grafana
+    - 'libxcb'  # for grafana
+    - 'libXcursor'  # for grafana
+    - 'libXt'  # for grafana
+    - 'logrotate'
+    - 'net-tools'
+    - 'nfs-utils'
+    - 'nmap-ncat'
 
-  # Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package)
-  - 'opendistro-alerting-1.13.1.*'
-  - 'opendistro-index-management-1.13.1.*'
-  - 'opendistro-job-scheduler-1.13.0.*'
-  - 'opendistro-performance-analyzer-1.13.0.*'
-  - 'opendistro-security-1.13.1.*'
-  - 'opendistro-sql-1.13.0.*'
-  - 'opendistroforelasticsearch-kibana-1.13.1'  # kibana has shorter version
-  - 'openssl'
-  - 'perl'  # for vim
-  - 'perl-Getopt-Long'  # for vim
-  - 'perl-libs'  # for vim
-  - 'perl-Pod-Perldoc'  # for vim
-  - 'perl-Pod-Simple'  # for vim
-  - 'perl-Pod-Usage'  # for vim
-  - 'pgaudit15_13-1.5.0'
-  - 'policycoreutils-python'  # for container-selinux
-  - 'pyldb'  # for cifs-utils
-  - 'python-cffi'  # for python2-cryptography
-  - 'python-firewall'  # for firewalld
-  - 'python-kitchen'  # for yum-utils
-  - 'python-lxml'  # for java-1.8.0-openjdk-headless
-  - 'python-psycopg2'
-  - 'python-pycparser'  # for python2-cryptography
-  - 'python-setuptools'
-  - 'python-slip-dbus'  # for firewalld
-  - 'python2-cryptography'  # for Ansible (certificate modules)
-  - 'python3-3.6.8'
-  - 'quota'  # for nfs-utils
-  - 'rabbitmq-server-3.8.9'
-  - 'rh-haproxy18'
-  - 'rh-haproxy18-haproxy-syspaths'
-  - 'postgresql13-server'
-  - 'repmgr10-5.2.1'  # used to upgrade repmgr first
-  - 'repmgr13-5.2.1'
-  - 'samba-client'
-  - 'samba-client-libs'  # for samba-client
-  - 'samba-common'
-  - 'samba-libs'  # for cifs-utils
-  - 'sysstat'
-  - 'tar'
-  - 'telnet'
-  - 'tmux'
-  - 'urw-base35-fonts'  # for grafana
-  - 'unzip'
-  - 'vim-common'  # for vim
-  - 'vim-enhanced'
-  - 'wget'
-  - 'xorg-x11-font-utils'  # for grafana
-  - 'xorg-x11-server-utils'  # for grafana
-  - 'yum-plugin-versionlock'
-  - 'yum-utils'
+    # Open Distro for Elasticsearch plugins are installed individually to not download them twice in different versions (as dependencies of opendistroforelasticsearch package)
+    - 'opendistro-alerting-1.13.1.*'
+    - 'opendistro-index-management-1.13.1.*'
+    - 'opendistro-job-scheduler-1.13.0.*'
+    - 'opendistro-performance-analyzer-1.13.0.*'
+    - 'opendistro-security-1.13.1.*'
+    - 'opendistro-sql-1.13.0.*'
+    - 'opendistroforelasticsearch-kibana-1.13.1'  # kibana has shorter version
+    - 'openssl'
+    - 'perl'  # for vim
+    - 'perl-Getopt-Long'  # for vim
+    - 'perl-libs'  # for vim
+    - 'perl-Pod-Perldoc'  # for vim
+    - 'perl-Pod-Simple'  # for vim
+    - 'perl-Pod-Usage'  # for vim
+    - 'pgaudit15_13-1.5.0'
+    - 'policycoreutils-python'  # for container-selinux
+    - 'pyldb'  # for cifs-utils
+    - 'python-cffi'  # for python2-cryptography
+    - 'python-firewall'  # for firewalld
+    - 'python-kitchen'  # for yum-utils
+    - 'python-lxml'  # for java-1.8.0-openjdk-headless
+    - 'python-psycopg2'
+    - 'python-pycparser'  # for python2-cryptography
+    - 'python-setuptools'
+    - 'python-slip-dbus'  # for firewalld
+    - 'python2-cryptography'  # for Ansible (certificate modules)
+    - 'python3-3.6.8'
+    - 'quota'  # for nfs-utils
+    - 'rabbitmq-server-3.8.9'
+    - 'rh-haproxy18'
+    - 'rh-haproxy18-haproxy-syspaths'
+    - 'postgresql13-server'
+    - 'repmgr10-5.2.1'  # used to upgrade repmgr first
+    - 'repmgr13-5.2.1'
+    - 'samba-client'
+    - 'samba-client-libs'  # for samba-client
+    - 'samba-common'
+    - 'samba-libs'  # for cifs-utils
+    - 'sysstat'
+    - 'tar'
+    - 'telnet'
+    - 'tmux'
+    - 'urw-base35-fonts'  # for grafana
+    - 'unzip'
+    - 'vim-common'  # for vim
+    - 'vim-enhanced'
+    - 'wget'
+    - 'xorg-x11-font-utils'  # for grafana
+    - 'xorg-x11-server-utils'  # for grafana
+    - 'yum-plugin-versionlock'
+    - 'yum-utils'
+
+    # to make remote-to-remote "synchronize" work in ansible
+    - 'rsync'
 
-  # to make remote-to-remote "synchronize" work in ansible
-  - 'rsync'
+    # K8s v1.18.6 (Epiphany >= v0.7.1)
+    - 'kubeadm-1.18.6'
+    - 'kubectl-1.18.6'
+    - 'kubelet-1.18.6'
 
-  # K8s v1.18.6 (Epiphany >= v0.7.1)
-  - 'kubeadm-1.18.6'
-  - 'kubectl-1.18.6'
-  - 'kubelet-1.18.6'
+    # K8s v1.19.15 (Epiphany >= v1.3 transitional version)
+    - 'kubeadm-1.19.15'
+    - 'kubectl-1.19.15'
+    - 'kubelet-1.19.15'
 
-  # K8s v1.19.15 (Epiphany >= v1.3 transitional version)
-  - 'kubeadm-1.19.15'
-  - 'kubectl-1.19.15'
-  - 'kubelet-1.19.15'
+    # K8s v1.20.12
+    - 'kubeadm-1.20.12'
+    - 'kubectl-1.20.12'
+    - 'kubelet-1.20.12'
 
-  # K8s v1.20.12
-  - 'kubeadm-1.20.12'
-  - 'kubectl-1.20.12'
-  - 'kubelet-1.20.12'
+    # K8s v1.21.7 (Epiphany >= v1.3, transitional version)
+    - 'kubeadm-1.21.7'
+    - 'kubectl-1.21.7'
+    - 'kubelet-1.21.7'
 
-  # K8s v1.21.7 (Epiphany >= v1.3, transitional version)
-  - 'kubeadm-1.21.7'
-  - 'kubectl-1.21.7'
-  - 'kubelet-1.21.7'
+    # K8s v1.22.4
+    - 'kubeadm-1.22.4'
+    - 'kubectl-1.22.4'
+    - 'kubelet-1.22.4'
 
-  # K8s v1.22.4
-  - 'kubeadm-1.22.4'
-  - 'kubectl-1.22.4'
-  - 'kubelet-1.22.4'
+    # Kubernetes Generic
+    - 'kubernetes-cni-0.8.6-0'  # since K8s v1.18.6
+    - 'kubernetes-cni-0.8.7-0'  # since K8s v1.19.15
 
-  # Kubernetes Generic
-  - 'kubernetes-cni-0.8.6-0'  # since K8s v1.18.6
-  - 'kubernetes-cni-0.8.7-0'  # since K8s v1.19.15
+  from_url:
+    # Github repository for erlang rpm is used since packagecloud repository is limited to a certain number of versions and erlang package from erlang-solutions repository is much more complex and bigger
+    'https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1.el7.x86_64.rpm':
+      sha256: c336ef007a027aebb6975cadcb134e541fb6f34b17699f067186d180246d1a51
+    # Grafana package is not downloaded from repository since it was not reliable (issue #2449)
+    'https://dl.grafana.com/oss/release/grafana-8.3.2-1.x86_64.rpm':
+      sha256: 5ad3cc7137bb8599f21eca40d08d671f3c508d856aea01b5a29576fda7f3c827