Implement new download requirements script (hitachienergy#805)

* rsync added to the Dockerfile

* add rsync to devcontainer

* files, images and crane moved to separate files

* added yaml

.devcontainer/Dockerfile

@@ -17,7 +17,7 @@ RUN : INSTALL APT REQUIREMENTS \
     jq libc6-dev libcap2-bin libffi-dev lsb-release \
     make musl-dev openssh-client procps \
     psmisc ruby-full sudo tar \
-    unzip vim \
+    unzip vim rsync \
  && apt-get -q autoremove -y \
  && apt-get -q clean -y \
  && rm -rf /var/lib/apt/lists/*

Dockerfile

@@ -14,7 +14,7 @@ COPY . /epicli
 RUN : INSTALL APT REQUIREMENTS \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
-        autossh curl gcc jq libcap2-bin libc6-dev libffi-dev make musl-dev openssh-client procps psmisc ruby-full sudo tar unzip vim \
+        autossh curl gcc jq libcap2-bin libc6-dev libffi-dev make musl-dev openssh-client procps psmisc ruby-full sudo tar unzip vim rsync \
 \
     && : INSTALL HELM BINARY \
     && curl -fsSLO https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \

ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py

@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+import datetime
+import logging
+import sys
+from os import getuid
+from typing import Dict, List
+
+from src.config import Config, OSType
+from src.error import CriticalError
+from src.mode.base_mode import BaseMode
+from src.mode.debian_family_mode import DebianFamilyMode
+from src.mode.red_hat_family_mode import RedHatFamilyMode
+
+
+MODES: Dict[OSType, BaseMode] = {
+    OSType.Ubuntu: DebianFamilyMode,
+    OSType.RedHat: RedHatFamilyMode,
+    OSType.CentOS: RedHatFamilyMode
+}
+
+
+def main(argv: List[str]) -> int:
+    try:
+        time_begin = datetime.datetime.now()
+        if getuid() != 0:
+            print('Error: Needs to be run as root!')
+            return 1
+
+        config = Config(argv)
+
+        MODES[config.os_type](config).run()
+
+        time_end = datetime.datetime.now() - time_begin
+        logging.info(f'Total execution time: {str(time_end).split(".")[0]}')
+    except CriticalError:
+        return 1
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

ansible/playbooks/roles/repository/files/download-requirements/repositories/x86_64/ubuntu.yml

@@ -0,0 +1,53 @@
+---
+repositories:
+  elastic_6:
+    key: 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+    content: 'deb https://artifacts.elastic.co/packages/oss-6.x/apt stable main'
+    path: '/etc/apt/sources.list.d/elastic-6.x.list'
+
+  kubernetes':
+    key: 'https://packages.cloud.google.com/apt/doc/apt-key.gpg'
+    content: 'deb http://apt.kubernetes.io/ kubernetes-xenial main'
+    path: '/etc/apt/sources.list.d/kubernetes.list'
+
+  erlang_solutions':
+    key: 'https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc'
+    content: 'deb https://packages.erlang-solutions.com/ubuntu focal contrib'
+    path: '/etc/apt/sources.list.d/erlang-23.x.list'
+
+  rabbitmq_server':
+    key: 'https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey'
+    content: 'deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu bionic main'
+    path: '/etc/apt/sources.list.d/rabbitmq.list'
+
+  docker_ce':
+    key: 'https://download.docker.com/linux/ubuntu/gpg'
+    content: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable'
+    path: '/etc/apt/sources.list.d/docker-ce.list'
+
+  elastic_7':
+    key: 'https://artifacts.elastic.co/GPG-KEY-elasticsearch'
+    content: 'deb https://artifacts.elastic.co/packages/oss-7.x/apt stable main'
+    path: '/etc/apt/sources.list.d/elastic-7.x.list'
+
+  opendistroforelasticsearch':
+    key: 'https://d3g5vo6xdbdb9a.cloudfront.net/GPG-KEY-opendistroforelasticsearch'
+    content: 'deb https://d3g5vo6xdbdb9a.cloudfront.net/apt stable main'
+    path: '/etc/apt/sources.list.d/opendistroforelasticsearch.list'
+
+  postgresql':
+    key: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+    content: 'deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main'
+    path: '/etc/apt/sources.list.d/pgdg.list'
+
+  # Historical packages from apt.postgresql.org
+  postgresql-archive':
+    key: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+    content: 'deb http://apt-archive.postgresql.org/pub/repos/apt focal-pgdg-archive main'
+    path: '/etc/apt/sources.list.d/pgdg-archive.list'
+
+  # Provides repmgr
+  2ndquadrant':
+    key: 'https://dl.2ndquadrant.com/gpg-key.asc'
+    content: 'deb https://dl.2ndquadrant.com/default/release/apt focal-2ndquadrant main'
+    path: '/etc/apt/sources.list.d/2ndquadrant-dl-default-release.list'

ansible/playbooks/roles/repository/files/download-requirements/requirements/crane.yml

@@ -0,0 +1,4 @@
+---
+crane:
+  - url: 'https://github.com/google/go-containerregistry/releases/download/v0.4.1/go-containerregistry_Linux_x86_64.tar.gz'
+    sha256: def1364f9483d133ccc6b1c4876f59a653d024c8866d96ecda026561d38c349b

ansible/playbooks/roles/repository/files/download-requirements/requirements/dashboards.yml

@@ -0,0 +1,66 @@
+---
+dashboards:
+  # Kubernetes Cluster
+  - name: grafana_dashboard_7249
+    url: 'https://grafana.com/api/dashboards/7249/revisions/1/download'
+    sha256: 41cc2794b1cc9fc537baf045fee12d086d23632b4c8b2e88985274bb9862e731
+
+  # Kubernetes cluster monitoring (via Prometheus)
+  - name: grafana_dashboard_315
+    url: 'https://grafana.com/api/dashboards/315/revisions/3/download'
+    sha256: ee46dd6e68a9950aa78e8c88ae5e565c8ebde6cbdbe08972a70f06c5486618fb
+
+  # Node Exporter for Prometheus
+  - name: grafana_dashboard_11074
+    url: 'https://grafana.com/api/dashboards/11074/revisions/9/download'
+    sha256: 151b23305da46eab84930e99175e1c07e375af73dbbb4b8f501ca25f5ac62785
+
+  # Node Exporter Server Metrics
+  - name: grafana_dashboard_405
+    url: 'https://grafana.com/api/dashboards/405/revisions/8/download'
+    sha256: 97675027cbd5b7241e93a2b598654c4b466bc909eeb6358ba123d500094d913c
+
+  # Postgres Overview
+  - name: grafana_dashboard_455
+    url: 'https://grafana.com/api/dashboards/455/revisions/2/download'
+    sha256: c66b91ab8d258b0dc005d3ee4dac3a5634a627c79cc8053875f76ab1e369a362
+
+  # PostgreSQL Database
+  - name: grafana_dashboard_9628
+    url: 'https://grafana.com/api/dashboards/9628/revisions/7/download'
+    sha256: c64cc38ad9ebd7af09551ee83e669a38f62a76e7c80929af5668a5852732b376
+
+  # RabbitMQ Monitoring
+  - name: grafana_dashboard_4279
+    url: 'https://grafana.com/api/dashboards/4279/revisions/4/download'
+    sha256: 74d47be868da52c145240ab5586d91ace9e9218ca775af988f9d60e501907a25
+
+  # Node Exporter Full
+  - name: grafana_dashboard_1860
+    url: 'https://grafana.com/api/dashboards/1860/revisions/23/download'
+    sha256: 225faab8bf35c1723af14d4c069882ccb92b455d1941c6b1cf3d95a1576c13d7
+
+  # Kafka Exporter Overview
+  - name: grafana_dashboard_7589
+    url: 'https://grafana.com/api/dashboards/7589/revisions/5/download'
+    sha256: cf020e14465626360418e8b5746818c80d77c0301422f3060879fddc099c2151
+
+  # HaProxy backend (or frontend/servers)
+  - name: grafana_dashboard_789
+    url: 'https://grafana.com/api/dashboards/789/revisions/1/download'
+    sha256: 6a9b4bdc386062287af4f7d56781103a2e45a51813596a65f03c1ae1d4d3e919
+
+  # Docker and Host Monitoring w/ Prometheus
+  - name: grafana_dashboard_179
+    url: 'https://grafana.com/api/dashboards/179/revisions/7/download'
+    sha256: 8d67350ff74e715fb1463f2406f24a73377357d90344f8200dad9d1b2a8133c2
+
+  # Kubernetes pod and cluster monitoring (via Prometheus)
+  - name: grafana_dashboard_6663
+    url: 'https://grafana.com/api/dashboards/6663/revisions/1/download'
+    sha256: d544d88069e1b793ff3d8f6970df641ad9a66217e69b629621e1ecbb2f06aa05
+
+  # RabbitMQ cluster monitoring (via Prometheus)
+  - name: grafana_dashboard_10991
+    url: 'https://grafana.com/api/dashboards/10991/revisions/11/download'
+    sha256: 66340fa3256d432287cba75ab5177eb058c77afa7d521a75d58099f95b1bff50

ansible/playbooks/roles/repository/files/download-requirements/requirements/files.yml

@@ -0,0 +1,40 @@
+---
+files:
+  # --- Exporters ---
+  - url: 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz'
+    sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0
+
+  - url: 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar'
+    sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e
+
+  - url: 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz'
+    sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949
+
+  - url: 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz'
+    sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1
+
+  # --- Misc ---
+  - url: 'https://archive.apache.org/dist/kafka/2.6.0/kafka_2.12-2.6.0.tgz'
+    sha256: 086bf9ca1fcbe2abe5c62e73d6f172adb1ee5a5b42732e153fb4d4ec82dab69f
+
+  - url: 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz'
+    sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3
+
+  - url: 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz'
+    sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc
+
+  - url: 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz'
+    sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3
+
+  - url: 'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz'
+    sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457
+
+  - url: 'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz'
+    sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961
+
+  # --- Helm charts ---
+  - url: 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz'
+    sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f
+
+  - url: 'https://helm.elastic.co/helm/filebeat/filebeat-7.9.2.tgz'
+    sha256: 5140b4c4473ca33a0af4c3f70545dcc89735c0a179d974ebc150f1f28ac229ab

ansible/playbooks/roles/repository/files/download-requirements/requirements/images.yml

@@ -0,0 +1,158 @@
+---
+images:
+  - name: 'haproxy:2.2.2-alpine'
+    sha256: f42019aaa7b8d10a7818a72ecde001b91e83df1d9ad0d1e38805968791880a75
+
+  - name: 'kubernetesui/dashboard:v2.3.1'
+    sha256: 1351ee7605362e7fcb84a645cf81c7ae915ce1034de9d9cf00f1246882976f59
+
+  - name: 'kubernetesui/metrics-scraper:v1.0.7'
+    sha256: 4fde72c37aa57868f19a5552b89eab71226eccec95fa33801146577880739050
+
+  - name: 'registry:2'
+    sha256: bdd3efacf7f73edbca2da6fae2cb46adc0a48999dbbb9f362e56f43187a3e34c
+
+  # applications
+  - name: 'bitnami/pgpool:4.2.4'
+    sha256: 3388e83ef861545f363c674b516e58c67dda0e15adf208ed5dee627e4a9a6453
+
+  - name: 'bitnami/pgbouncer:1.16.0'
+    sha256: f66d8c29402f03180393cea7bc1a75b708807113a77877e85405fcca1f3f2b0e
+
+  - name: 'epiphanyplatform/keycloak:14.0.0'
+    sha256: a6c143ab8f2e53f0af73f068cda0d8bd50aa792a6729f901367ace44a6bedf58
+
+  - name: 'rabbitmq:3.8.9'
+    sha256: 7f28ecc2db56fb02fa237adc8cef811352ebde18365741b0be6da8d850465e59
+
+  # K8s
+  # v1.18.6
+  - name: 'k8s.gcr.io/kube-apiserver:v1.18.6'
+    sha256: 46a6a5db199cf175ddf83dc4f8c4bca9ef852b36ffd8929dbc793a9a5ac7f42e
+
+  - name: 'k8s.gcr.io/kube-controller-manager:v1.18.6'
+    sha256: 066153685539eb43de5702d8924ed9c7c449a1a6b4557b0aead5e8c80850f5f7
+
+  - name: 'k8s.gcr.io/kube-scheduler:v1.18.6'
+    sha256: 257187a3ff641276ad174ef2a01176f4655997e235c83ae5f27a96a3ed5ff01b
+
+  - name: 'k8s.gcr.io/kube-proxy:v1.18.6'
+    sha256: 6fa027ea9c16e230384617fbfd3b6ba0d70c6f4783fe8b23af03d226ebcb444b
+
+  - name: 'k8s.gcr.io/coredns:1.6.7'
+    sha256: bc895982dfec04707553353fef5e6f35b8dfc2ec58a4bc67d234eee5dbed814f
+
+  - name: 'k8s.gcr.io/etcd:3.4.3-0'
+    sha256: 2770a10d5b98167c07b9326167a52b3d58bf9e9241d813ed9ec88472e958e12a
+
+  - name: 'quay.io/coreos/flannel:v0.12.0-amd64'
+    sha256: 0fd9251d0bed2e9fba12ef21c3443acf461fb7d0d9e469ff7f0ed51f68848bec
+
+  - name: 'quay.io/coreos/flannel:v0.12.0'
+    sha256: 14c9a68e0038d764992cf06119e67f86e1d1e4489d1c0bac0ee2a69ff484ab44
+
+  - name: 'calico/cni:v3.15.0'
+    sha256: 1007ddea91576ff50325e6a8dad83ba7596e7369c5c5f2318118438e029a348a
+
+  - name: 'calico/kube-controllers:v3.15.0'
+    sha256: 4e13dd977d99a9c846b740531a9e4210ddb7cb5be4b8c05264aad2f541702e28
+
+  - name: 'calico/node:v3.15.0'
+    sha256: 8aedb15067f694aa1220589c1cb3e51a3a652d127143a55a41b9be28b0ea7da7
+
+  - name: 'calico/pod2daemon-flexvol:v3.15.0'
+    sha256: 4cf6c67db34bdc5598e28fc54429afa0362b299fdc6da064335c6bff8e01ca1a
+
+  # v1.19.15
+  - name: 'k8s.gcr.io/kube-apiserver:v1.19.15'
+    sha256: 5a128204ea5aeae8eb84cfc7cf759b6e31c97eb0fc57652d10199d6406ab7333
+
+  - name: 'k8s.gcr.io/kube-controller-manager:v1.19.15'
+    sha256: abdd9350d664f4a235d3a300fa55f9a4a6c5a26d299d7e0ec0e11476be589aea
+
+  - name: 'k8s.gcr.io/kube-scheduler:v1.19.15'
+    sha256: 08732fd863f4127dd34692fabc2f9e023f6d6d2c238eb34e928cf3ef9c16f0cf
+
+  - name: 'k8s.gcr.io/kube-proxy:v1.19.15'
+    sha256: 131b72a5d153c153bbc85e5cfbef7fbdc0ff7f43ffb4e9555b1a95b8625f0226
+
+  # v1.20.12
+  - name: 'k8s.gcr.io/kube-apiserver:v1.20.12'
+    sha256: 618e808f0b2cc5203c4105f417817a8a5cf10a6829493fdc1a5f9c7d3bd45520
+
+  - name: 'k8s.gcr.io/kube-controller-manager:v1.20.12'
+    sha256: 30c5431215d61593b0debf5c2a2e3b818f8af83689c55a06817f4f23cf950f70
+
+  - name: 'k8s.gcr.io/kube-scheduler:v1.20.12'
+    sha256: f31e3de525f75953ab3494df4879352c4aaf0f7bd95e3960c29e52dfd81783d0
+
+  - name: 'k8s.gcr.io/kube-proxy:v1.20.12'
+    sha256: 40bb2f9e229e846034dde56e41a125b14c697be96102dc3b02f60d2b16b5fb4c
+
+  - name: 'k8s.gcr.io/coredns:1.7.0'
+    sha256: 08368ce06f22002639ffe095a2b2a0d16da50b2a7e821206cb97957d183dde92
+
+  - name: 'k8s.gcr.io/pause:3.2'
+    sha256: 64bb6a32f297d03e8b570e58d7dd00a171ed5686295d966b2fd583165fb93f9e
+
+  # v1.21.7
+  - name: 'k8s.gcr.io/kube-apiserver:v1.21.7'
+    sha256: 903a9db6809afa0fd6a35109102916c5648518de931b424e87173dfcfbc98f24
+
+  - name: 'k8s.gcr.io/kube-controller-manager:v1.21.7'
+    sha256: e7b9752b044582dc841f7f0ef538835712ca1e5311a1ee667464be2c361c768d
+
+  - name: 'k8s.gcr.io/kube-scheduler:v1.21.7'
+    sha256: 3c3585da6d26cffa725b40b8c49b4d0720485320144bcdc51013f546cc781f12
+
+  - name: 'k8s.gcr.io/kube-proxy:v1.21.7'
+    sha256: 31b1f8a34852b9a66a174df742346927d2ffcf16366ebbc22e71d2fcb01b1b42
+
+  - name: 'k8s.gcr.io/coredns/coredns:v1.8.0'
+    sha256: 6bf705594c7676ab6f8fcf22876fe5d33eb71018928e73fff90255b460aded6a
+
+  - name: 'k8s.gcr.io/etcd:3.4.13-0'
+    sha256: 377554d89698f6c4e7a1aafaa756ad87bcefdc45ea33123fb9945501623aa501
+
+  - name: 'k8s.gcr.io/pause:3.4.1'
+    sha256: d8f6de1afe05ef0de2f25368599e9b540da19bbf7a83746d6fbed06bfef4edac
+
+  # v1.22.4
+  - name: 'k8s.gcr.io/kube-apiserver:v1.22.4'
+    sha256: 86c7114f56570eccdf1542eb08b1c17e1d453e56c028795a3751ee39d780d2e7
+
+  - name: 'k8s.gcr.io/kube-controller-manager:v1.22.4'
+    sha256: 3651fbadbbd1785244862f6f0ee9e29d267676e1536db3dd89bae3be5a25145e
+
+  - name: 'k8s.gcr.io/kube-scheduler:v1.22.4'
+    sha256: 129e8dada97246067eede38a2f0634428ee77d05d7a2c38c84258e1977e7a7e6
+
+  - name: 'k8s.gcr.io/kube-proxy:v1.22.4'
+    sha256: a5331a2d1b22487cad455d2e6e82e107ecd7e5124a5bf9c6d9044c99231d7c55
+
+  - name: 'k8s.gcr.io/coredns/coredns:v1.8.4'
+    sha256: 0db432fb9f3de461a3fbc463628aaef4609f179a55a9ae487b2f1f1af7673939
+
+  - name: 'k8s.gcr.io/etcd:3.5.0-0'
+    sha256: b56aff902afec396f75c9f7e8db6b47e1da041730848cd38e69db4ff74b72dfc
+
+  - name: 'k8s.gcr.io/pause:3.5'
+    sha256: 44269aea9e3efb259c9bd305569e7bd25805958ebb6f1c7f758101d311b9493a
+
+  - name: 'quay.io/coreos/flannel:v0.14.0-amd64'
+    sha256: bb95008a5b6af496db5ebd120542035952c559eec5b7414ce635aea5cc8592c1
+
+  - name: 'quay.io/coreos/flannel:v0.14.0'
+    sha256: d4c7f16b7f39dc4cfa4226a04514808c279aa01863886df0f32d27a93fef76c2
+
+  - name: 'calico/cni:v3.20.3'
+    sha256: 289009d818d6d676ea509f0baf717dacddec81d2b64cb0c55450f6e881f90432
+
+  - name: 'calico/kube-controllers:v3.20.3'
+    sha256: 2c4146dd95d62924b727d84a62e9ec306647a702da7715d6889021c2c9e07a2d
+
+  - name: 'calico/node:v3.20.3'
+    sha256: 267511105a133492f845dd94a520b319003ca26d404cabe251a3256d11dfd84c
+
+  - name: 'calico/pod2daemon-flexvol:v3.20.3'
+    sha256: 5b9e837c43575b4bde353f1e9c1e2ce45c399a192b8e72a622dace55567feee7