From 022954982b3b2ce198f893bd50590e82a66b65ac Mon Sep 17 00:00:00 2001 From: sbbroot <86356638+sbbroot@users.noreply.github.com> Date: Tue, 28 Jun 2022 16:16:20 +0200 Subject: [PATCH] [2.0.1] Adaptive mode for downloading requirements (#3188) * Split available_roles and roles_mapping into separate yaml documents (#3097) (#3119) * available_roles splitted into feature-mappings and features documents * feature-mappings added to the Init by default * Add manifest file parsing (#3105) (#3130) * Add `-m/--manifest` flag to accept manifest.yml produced by `epicli init/prepare` * Add `-v/--verbose` mode for printing out parsed manifest data * Add ManifestReader class used for paring the manifest.yml file * Move src/command/*.py to debian/redhat subdirs where needed * Optimize Grafana dashboards downloading (#3131) (#3150) * Optimize files downloading (#3116) (#3156) * Add image-registry configuration reading (#3106) (#3159) --- .ansible-lint | 1 + ansible/playbooks/filter_plugins/container.py | 27 + ansible/playbooks/repository.yml | 2 +- .../roles/download/tasks/list_files.yml | 28 +- .../roles/download/tasks/list_images.yml | 9 + .../download/tasks/list_requirements.yml | 25 + .../roles/image_registry/tasks/main.yml | 47 +- .../roles/opensearch/defaults/main.yml | 1 - .../roles/repository/defaults/main.yml | 3 +- .../download-requirements.py | 4 +- .../requirements/aarch64/files.yml | 16 +- .../requirements/aarch64/images.yml | 88 ++-- .../requirements/x86_64/files.yml | 19 +- .../requirements/x86_64/images.yml | 263 +++++----- .../src/command/{ => debian}/apt.py | 0 .../src/command/{ => debian}/apt_cache.py | 0 .../src/command/{ => debian}/apt_key.py | 0 .../src/command/{ => redhat}/dnf.py | 18 +- .../{ => redhat}/dnf_config_manager.py | 0 .../src/command/{ => redhat}/dnf_download.py | 2 +- .../src/command/{ => redhat}/dnf_repoquery.py | 0 .../src/command/{ => redhat}/rpm.py | 0 .../src/command/toolchain.py | 16 +- .../src/config/config.py | 156 +++++- .../src/config/manifest_reader.py | 117 +++++ .../src/config/version.py | 24 + .../files/download-requirements/src/error.py | 27 +- .../src/mode/base_mode.py | 36 +- .../tests/command/{ => debian}/test_apt.py | 2 +- .../command/{ => debian}/test_apt_cache.py | 2 +- .../command/{ => debian}/test_apt_key.py | 2 +- .../tests/command/{ => redhat}/test_dnf.py | 4 +- .../command/{ => redhat}/test_dnf_base.py | 2 +- .../{ => redhat}/test_dnf_config_manager.py | 2 +- .../command/{ => redhat}/test_dnf_download.py | 2 +- .../{ => redhat}/test_dnf_repoquery.py | 2 +- .../tests/command/{ => redhat}/test_rpm.py | 2 +- .../tests/config/test_config.py | 75 +++ .../tests/config/test_manifest_reader.py | 26 + .../tests/config/test_version.py | 17 + .../tests/data/config.py | 475 ++++++++++++++++++ .../tests/data/manifest_reader.py | 438 ++++++++++++++++ .../repository/tasks/clean-up-epirepo.yml | 34 +- .../tasks/copy-download-requirements.yml | 6 + .../tasks/download-requirements.yml | 23 +- .../roles/repository/tasks/setup.yml | 4 +- cli/epicli.py | 4 + cli/src/Config.py | 19 + cli/src/ansible/AnsibleInventoryCreator.py | 8 +- cli/src/ansible/AnsibleVarsGenerator.py | 12 +- cli/src/commands/Apply.py | 36 +- cli/src/commands/Init.py | 3 + cli/src/commands/Upgrade.py | 5 + cli/src/schema/ConfigurationAppender.py | 75 +-- docs/changelogs/CHANGELOG-2.0.md | 12 +- docs/home/howto/CLUSTER.md | 67 +-- docs/home/howto/DATABASES.md | 6 +- docs/home/howto/KUBERNETES.md | 8 +- docs/home/howto/MODULES.md | 403 +++++++++++++++ docs/home/howto/UPGRADE.md | 6 +- ...ature-mapping.yml => feature-mappings.yml} | 56 +-- .../defaults/configuration/features.yml | 54 ++ .../defaults/configuration/image-registry.yml | 475 +++++++++--------- ...ature-mapping.yml => feature-mappings.yml} | 15 +- .../validation/configuration/features.yml | 15 + .../configuration/image-registry.yml | 104 ++-- schema/common/validation/epiphany-cluster.yml | 4 +- 67 files changed, 2717 insertions(+), 717 deletions(-) create mode 100644 ansible/playbooks/filter_plugins/container.py create mode 100644 ansible/playbooks/roles/download/tasks/list_images.yml create mode 100644 ansible/playbooks/roles/download/tasks/list_requirements.yml rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt_cache.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => debian}/apt_key.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf.py (93%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_config_manager.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_download.py (96%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/dnf_repoquery.py (100%) rename ansible/playbooks/roles/repository/files/download-requirements/src/command/{ => redhat}/rpm.py (100%) create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/src/config/version.py rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt.py (97%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt_cache.py (93%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => debian}/test_apt_key.py (88%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf.py (97%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_base.py (86%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_config_manager.py (93%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_download.py (94%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_dnf_repoquery.py (96%) rename ansible/playbooks/roles/repository/files/download-requirements/tests/command/{ => redhat}/test_rpm.py (96%) create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_version.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py create mode 100644 ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py create mode 100644 docs/home/howto/MODULES.md rename schema/common/defaults/configuration/{feature-mapping.yml => feature-mappings.yml} (57%) create mode 100644 schema/common/defaults/configuration/features.yml rename schema/common/validation/configuration/{feature-mapping.yml => feature-mappings.yml} (78%) create mode 100644 schema/common/validation/configuration/features.yml diff --git a/.ansible-lint b/.ansible-lint index 92478d5e18..30affe3883 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -38,6 +38,7 @@ skip_list: - meta-no-info - package-latest - fqcn-builtins + - no-jinja-when ################## # Tags to follow # diff --git a/ansible/playbooks/filter_plugins/container.py b/ansible/playbooks/filter_plugins/container.py new file mode 100644 index 0000000000..8aaae8d8a4 --- /dev/null +++ b/ansible/playbooks/filter_plugins/container.py @@ -0,0 +1,27 @@ +from typing import Any, Dict, List + + +class FilterModule: + """ Filters for Python's container types """ + + def filters(self): + return { + 'dict_to_list': self.dict_to_list + } + + def dict_to_list(self, data: Dict, only_values: bool = False, only_keys: bool = False) -> List: + """ + Convert dict to list without using Ansible's loop mechanism with dict2items filter. + + :param data: to be converted into a list + :param only_values: construct list with only dict's values + :param only_keys: construct list with only dict's keys + :return: data transformed into a list + """ + if only_values: + return list(data.values()) + + if only_keys: + return list(data.keys()) + + return list(data.items()) diff --git a/ansible/playbooks/repository.yml b/ansible/playbooks/repository.yml index b76cdda5fb..7a7a5e55a4 100644 --- a/ansible/playbooks/repository.yml +++ b/ansible/playbooks/repository.yml @@ -1,5 +1,5 @@ --- -# This playbook is empty by purpose, just to enable repository role in configuration/feature-mapping +# This playbook is empty by purpose, just to enable repository role in configuration/features # to populate defaults/configuration to Ansible vars - hosts: "!all" tasks: [] diff --git a/ansible/playbooks/roles/download/tasks/list_files.yml b/ansible/playbooks/roles/download/tasks/list_files.yml index 270fe93070..987e5adc52 100644 --- a/ansible/playbooks/roles/download/tasks/list_files.yml +++ b/ansible/playbooks/roles/download/tasks/list_files.yml @@ -1,25 +1,9 @@ --- -- name: Get file listing - uri: - method: GET - url: "{{ repository_url }}/files/?F=0" # F=0 formats the listing as a simple list (not FancyIndexed) - body_format: raw - return_content: true - validate_certs: "{{ validate_certs | default(false, true) | bool }}" # handling "undefined", "null", "empty" and "boolean" values all at once - register: uri_list_files - until: uri_list_files is success - retries: 3 - delay: 2 - become: false +- name: Get files list from the repository + include_tasks: list_requirements.yml + vars: + _requirements: files -# TODO: make it work with yaml or json (instead of html, sic!). -- name: Parse html response and return file listing +- name: Set files in repository as fact set_fact: - list_files_result: >- - {{ lines | select('match', regexp) - | reject('match', '.*Parent Directory.*') - | map('regex_replace', regexp, '\1') - | list }} - vars: - lines: "{{ uri_list_files.content.splitlines() }}" - regexp: '.*
  • - + {{ lines | select('match', regexp) + | reject('match', '.*Parent Directory.*') + | map('regex_replace', regexp, '\1') + | list }} + vars: + lines: "{{ uri_list_files.content.splitlines() }}" + regexp: '.*
  • - - {{ specification.images_to_load[ansible_architecture].generic + specification.images_to_load[ansible_architecture].current }} - legacy_images: "{{ specification.images_to_load[ansible_architecture].legacy }}" + current_schema_images: "{{ specification.images_to_load[ansible_architecture].current }}" + generic_schema_images: "{{ specification.images_to_load[ansible_architecture].generic }}" + legacy_schema_images: "{{ specification.images_to_load[ansible_architecture].legacy }}" + + - name: Initialize image facts + set_fact: + requested_images: [] + current_images: [] + generic_images: [] + legacy_images: [] + + - name: Set list of current images to be loaded/pushed + set_fact: + current_images: "{{ current_schema_images | dict_to_list(only_values='True') | flatten }}" + + - name: Set list of generic images to be loaded/pushed + set_fact: + generic_images: "{{ generic_schema_images | dict_to_list(only_values='True') | flatten }}" + + - name: Set list of legacy images to be loaded/pushed + set_fact: + legacy_images: "{{ legacy_schema_images | dict_to_list(only_values='True') | flatten }}" + + - name: Merge current and generic images + set_fact: + current_and_generic_images: >- + {{ current_images + generic_images }} + + - name: Get list of available images + include_role: + name: download + tasks_from: list_images.yml + + - name: Filter only requested images + set_fact: # gather only images listed in schema to avoid downloading unknown files + requested_images: "{{ requested_images + [item] }}" + when: "{{ item.file_name in list_images_result }}" + loop: "{{ current_and_generic_images }}" - name: Load generic and current version images vars: docker_image: "{{ item }}" include_tasks: load-image.yml - loop: "{{ generic_and_current_images }}" + loop: "{{ requested_images }}" - name: Push generic and current version images to registry vars: docker_image: "{{ item }}" new_image_tag: "{{ image_registry_address }}/{{ item.name }}" include_tasks: push-image.yml - loop: "{{ generic_and_current_images }}" + loop: "{{ requested_images }}" - name: Load legacy version images to registry when upgrading when: is_upgrade_run diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml index a1765a6b29..7d62757958 100644 --- a/ansible/playbooks/roles/opensearch/defaults/main.yml +++ b/ansible/playbooks/roles/opensearch/defaults/main.yml @@ -59,4 +59,3 @@ certificates: ports: http: 9200 transport: 9300 -log4j_file_name: apache-log4j-2.17.1-bin.tar.gz diff --git a/ansible/playbooks/roles/repository/defaults/main.yml b/ansible/playbooks/roles/repository/defaults/main.yml index 5a0a7db45d..95b9f60c6b 100644 --- a/ansible/playbooks/roles/repository/defaults/main.yml +++ b/ansible/playbooks/roles/repository/defaults/main.yml @@ -1,4 +1,5 @@ --- download_requirements_dir: "/var/tmp/epi-download-requirements" -download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py" download_requirements_flag: "{{ download_requirements_dir }}/download-requirements-done.flag" +download_requirements_manifest: "{{ download_requirements_dir }}/manifest.yml" +download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py b/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py index 4b8d1e55cd..5fabc5c7a6 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py @@ -9,7 +9,7 @@ from src.command.toolchain import TOOLCHAINS from src.config.config import Config -from src.error import DownloadRequirementsError +from src.error import DownloadRequirementsException def install_missing_modules(config: Config): @@ -79,7 +79,7 @@ def main(argv: List[str]) -> int: time_end = datetime.datetime.now() - time_begin logging.info(f'Total execution time: {str(time_end).split(".")[0]}') - except DownloadRequirementsError: + except DownloadRequirementsException: return 1 return 0 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml index 373c131449..3270a882a7 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml @@ -3,45 +3,55 @@ files: # --- Exporters --- 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-arm64.tar.gz': sha256: 95ff0c723f3cdb6967b54c0208a5d0e67ad59dc53c1907a401cb8a448e53ec96 + deps: [kafka-exporter] 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e + deps: [kafka] 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-arm64.tar.gz': sha256: f19f35175f87d41545fa7d4657e834e3a37c1fe69f3bf56bc031a256117764e7 + deps: [node-exporter] 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-arm64.tar.gz': sha256: 82a1a4e07c7140f8e55532dbbdfea3bbba33dafc7ef0a221601bb2fd5359ff03 + deps: [postgres-exporter] # --- Misc --- 'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz': sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98 + deps: [kafka] 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz': sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3 + deps: [zookeeper] 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz': sha256: afa44f350797032ceb714598900cfdddbf81d6ef03d2ecbfc0221cc2cb28a6b9 + deps: [prometheus] 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-arm64.tar.gz': sha256: a7b4694b96cbf38b63ca92d05a6d3a2cf6df50a85a4d2a3fe2d758a65dcbec3b + deps: [prometheus] 'https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz': sha256: cd11f0ed12a658f3b78392528814350a508d2c53d8da7f04145909e94bda10f1 - - 'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz': - sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961 + deps: [helm] # --- Helm charts --- 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz': sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f + deps: [kubernetes-master] 'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz': sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79 + deps: [kubernetes-master] # --- OpenSearch Bundle --- 'https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-arm64.tar.gz': sha256: 5e8cd13ad1831e4a286a54334505c16c43ce8e50981100eea4eb18f79d3e63a5 + deps: [logging, opensearch] 'https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-arm64.tar.gz': sha256: 1f668d98f4670f1b88f03b19d30b2cc44ec439a7b2edff1a48034717d594cfe1 + deps: [logging, opensearch] diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml index 226bcef293..3ccb7c6d82 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml @@ -1,62 +1,64 @@ --- images: - 'haproxy:2.2.2-alpine': - sha1: 2fd3bd554ee6e126a1b1e5055ed0349beac81ffc + haproxy: + 'haproxy:2.2.2-alpine': + sha1: 2fd3bd554ee6e126a1b1e5055ed0349beac81ffc - 'kubernetesui/dashboard:v2.3.1': - sha1: 5dc90f59af4643952d5a728213983e6f3d884895 + image-registry: + 'registry:2.8.0': + sha1: c6cdd738fbdd2efaef9588b439fc3a7c0c090368 + allow_mismatch: true - 'kubernetesui/metrics-scraper:v1.0.7': - sha1: 90b57b399e7ed44bad422e7d9572bfd6d737724a + applications: + 'epiphanyplatform/keycloak:14.0.0': + sha1: c73be9d38580fc9819bec8942e2d2569196547e1 - 'registry:2.8.0': - sha1: c6cdd738fbdd2efaef9588b439fc3a7c0c090368 - allow_mismatch: true + 'rabbitmq:3.8.9': + sha1: 854d06bae1ee7e3a94570e1ce104618430988f57 - # applications - 'epiphanyplatform/keycloak:14.0.0': - sha1: c73be9d38580fc9819bec8942e2d2569196547e1 + kubernetes-master: + 'kubernetesui/dashboard:v2.3.1': + sha1: 5dc90f59af4643952d5a728213983e6f3d884895 - 'rabbitmq:3.8.9': - sha1: 854d06bae1ee7e3a94570e1ce104618430988f57 + 'kubernetesui/metrics-scraper:v1.0.7': + sha1: 90b57b399e7ed44bad422e7d9572bfd6d737724a + # K8s + # v1.22.4 + 'k8s.gcr.io/kube-apiserver:v1.22.4': + sha1: 6e101cfa4384346b45701e6dda5591a41fa5776d - # K8s - # v1.22.4 - 'k8s.gcr.io/kube-apiserver:v1.22.4': - sha1: 6e101cfa4384346b45701e6dda5591a41fa5776d + 'k8s.gcr.io/kube-controller-manager:v1.22.4': + sha1: 6561280956af24f9547dabde5758a4091558e771 - 'k8s.gcr.io/kube-controller-manager:v1.22.4': - sha1: 6561280956af24f9547dabde5758a4091558e771 + 'k8s.gcr.io/kube-scheduler:v1.22.4': + sha1: e224852d58ab649f3145cae3ed4f2926e66117bf - 'k8s.gcr.io/kube-scheduler:v1.22.4': - sha1: e224852d58ab649f3145cae3ed4f2926e66117bf + 'k8s.gcr.io/kube-proxy:v1.22.4': + sha1: 5e5e4032f3f6464ede1a4a85854013d0801c8eff - 'k8s.gcr.io/kube-proxy:v1.22.4': - sha1: 5e5e4032f3f6464ede1a4a85854013d0801c8eff + 'k8s.gcr.io/coredns/coredns:v1.8.4': + sha1: e5d7de1974e8f331892d9587a5312d4cdda04bb2 - 'k8s.gcr.io/coredns/coredns:v1.8.4': - sha1: e5d7de1974e8f331892d9587a5312d4cdda04bb2 + 'k8s.gcr.io/etcd:3.5.0-0': + sha1: fb1975ba3fc696fa7530c0752d15abe8ea23e80d - 'k8s.gcr.io/etcd:3.5.0-0': - sha1: fb1975ba3fc696fa7530c0752d15abe8ea23e80d + 'k8s.gcr.io/pause:3.5': + sha1: 8f3be7cc532c25b01a2e5e0943b4d55bce0b0f1c - 'k8s.gcr.io/pause:3.5': - sha1: 8f3be7cc532c25b01a2e5e0943b4d55bce0b0f1c + 'quay.io/coreos/flannel:v0.14.0-arm64': + sha1: 342b806ab5da6133438f925e22ced802e0e56861 - 'quay.io/coreos/flannel:v0.14.0-arm64': - sha1: 342b806ab5da6133438f925e22ced802e0e56861 + 'quay.io/coreos/flannel:v0.14.0': + sha1: 098ec78af9bf3a70afbd1a9743ff352f72fb036d - 'quay.io/coreos/flannel:v0.14.0': - sha1: 098ec78af9bf3a70afbd1a9743ff352f72fb036d + 'calico/cni:v3.20.3': + sha1: 5338d670e01558dd053768fb3cdc39c8568d6f4c - 'calico/cni:v3.20.3': - sha1: 5338d670e01558dd053768fb3cdc39c8568d6f4c + 'calico/kube-controllers:v3.20.3': + sha1: 24615f8f9c0cac651401cc633ba4850b8f17e175 - 'calico/kube-controllers:v3.20.3': - sha1: 24615f8f9c0cac651401cc633ba4850b8f17e175 + 'calico/node:v3.20.3': + sha1: c0432602b9e8dd47d3c869759f210305d9d87d90 - 'calico/node:v3.20.3': - sha1: c0432602b9e8dd47d3c869759f210305d9d87d90 - - 'calico/pod2daemon-flexvol:v3.20.3': - sha1: 6e1e59ad1cf6af46d4ce8c8b16f4014337c22411 + 'calico/pod2daemon-flexvol:v3.20.3': + sha1: 6e1e59ad1cf6af46d4ce8c8b16f4014337c22411 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml index e582b3ef9d..09583356a1 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/files.yml @@ -3,54 +3,67 @@ files: # --- Exporters --- 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz': sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0 + deps: [kafka-exporter] 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e + deps: [kafka] 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz': sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949 + deps: [node-exporter] 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz': sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1 + deps: [postgres-exporter] # --- Misc --- 'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz': sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98 + deps: [kafka] 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz': sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3 + deps: [zookeeper] 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz': sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc + deps: [prometheus] 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz': sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3 + deps: [prometheus] 'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz': sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457 - - 'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz': - sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961 + deps: [helm] # --- Helm charts --- 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz': sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f + deps: [kubernetes-master] 'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz': sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79 + deps: [kubernetes-master] 'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz': sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14 + deps: [kubernetes-master] 'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz': sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16 + deps: [kubernetes-master] # --- OpenSearch Bundle --- 'https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz': sha256: d40f2696623b6766aa235997e2847a6c661a226815d4ba173292a219754bd8a8 + deps: [logging, opensearch] 'https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz': sha256: 14623798e61be6913e2a218d6ba3e308e5036359d7bda58482ad2f1340aa3c85 + deps: [opensearch-dashboards] 'https://github.com/opensearch-project/perftop/releases/download/1.2.0.0/opensearch-perf-top-1.2.0.0-linux-x64.zip': sha256: e8f9683976001a8cf59a9f86da5caafa10b88643315f0af2baa93a9354d41e2b + deps: [logging, opensearch] diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml index f7d48d5ccf..a9bb9ca941 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml @@ -1,191 +1,194 @@ --- images: - 'haproxy:2.2.2-alpine': - sha1: dff8993b065b7f7846adb553548bcdcfcd1b6e8e + haproxy: + 'haproxy:2.2.2-alpine': + sha1: dff8993b065b7f7846adb553548bcdcfcd1b6e8e - 'kubernetesui/dashboard:v2.3.1': - sha1: 8c8a4ac7a643f9c5dd9e5d22876c434187312db8 + image-registry: + 'registry:2.8.0': + sha1: 89795c17099199c752d02ad8797c1d4565a08aff + allow_mismatch: true - 'kubernetesui/metrics-scraper:v1.0.7': - sha1: 5a0052e2afd3eef3ae638be21938b29b1d608ebe + applications: + 'bitnami/pgpool:4.2.4': + sha1: 66741f3cf4a508bd1f80e2965b0086a4c0fc3580 - 'registry:2.8.0': - sha1: 89795c17099199c752d02ad8797c1d4565a08aff - allow_mismatch: true + 'bitnami/pgbouncer:1.16.0': + sha1: f2e37eecbf9aed44d5566f06dcc101c1ba9edff9 - # applications - 'bitnami/pgpool:4.2.4': - sha1: 66741f3cf4a508bd1f80e2965b0086a4c0fc3580 + 'epiphanyplatform/keycloak:14.0.0': + sha1: b59d75a967cedd3a4cf5867eced2fb5dff52f60e - 'bitnami/pgbouncer:1.16.0': - sha1: f2e37eecbf9aed44d5566f06dcc101c1ba9edff9 + 'rabbitmq:3.8.9': + sha1: c64408bf5bb522f47d5323652dd5e60560dcb5bc - 'epiphanyplatform/keycloak:14.0.0': - sha1: b59d75a967cedd3a4cf5867eced2fb5dff52f60e + kubernetes-master: + 'kubernetesui/dashboard:v2.3.1': + sha1: 8c8a4ac7a643f9c5dd9e5d22876c434187312db8 - 'rabbitmq:3.8.9': - sha1: c64408bf5bb522f47d5323652dd5e60560dcb5bc + 'kubernetesui/metrics-scraper:v1.0.7': + sha1: 5a0052e2afd3eef3ae638be21938b29b1d608ebe - # K8s - # v1.18.6 - 'k8s.gcr.io/kube-apiserver:v1.18.6': - sha1: 164968226f4617abaa31e6108ed9034a1e302f4f + # K8s + # v1.18.6 + 'k8s.gcr.io/kube-apiserver:v1.18.6': + sha1: 164968226f4617abaa31e6108ed9034a1e302f4f - 'k8s.gcr.io/kube-controller-manager:v1.18.6': - sha1: ebea3fecab9e5693d31438fa37dc4d02c6914d67 + 'k8s.gcr.io/kube-controller-manager:v1.18.6': + sha1: ebea3fecab9e5693d31438fa37dc4d02c6914d67 - 'k8s.gcr.io/kube-scheduler:v1.18.6': - sha1: 183d29c4fdcfda7478d08240934fdb6845e2e3ec + 'k8s.gcr.io/kube-scheduler:v1.18.6': + sha1: 183d29c4fdcfda7478d08240934fdb6845e2e3ec - 'k8s.gcr.io/kube-proxy:v1.18.6': - sha1: 62da886e36efff0c03a16e19c1442a1c3040fbf1 + 'k8s.gcr.io/kube-proxy:v1.18.6': + sha1: 62da886e36efff0c03a16e19c1442a1c3040fbf1 - 'k8s.gcr.io/coredns:1.6.7': - sha1: 76615ffabb22fd4fb3d562cb6ebcd243f8826e48 + 'k8s.gcr.io/coredns:1.6.7': + sha1: 76615ffabb22fd4fb3d562cb6ebcd243f8826e48 - 'k8s.gcr.io/etcd:3.4.3-0': - sha1: 6ee82ddb1bbc7f1831c42046612b8bcfbb171b45 + 'k8s.gcr.io/etcd:3.4.3-0': + sha1: 6ee82ddb1bbc7f1831c42046612b8bcfbb171b45 - 'quay.io/coreos/flannel:v0.12.0-amd64': - sha1: 3516522e779373983992095e61eb6615edd50d1f + 'quay.io/coreos/flannel:v0.12.0-amd64': + sha1: 3516522e779373983992095e61eb6615edd50d1f - 'quay.io/coreos/flannel:v0.12.0': - sha1: 2cb6ce8f1361886225526767c4a0422c039453c8 + 'quay.io/coreos/flannel:v0.12.0': + sha1: 2cb6ce8f1361886225526767c4a0422c039453c8 - 'calico/cni:v3.15.0': - sha1: aa59f624c223bc398a42c7ba9e628e8143718e58 + 'calico/cni:v3.15.0': + sha1: aa59f624c223bc398a42c7ba9e628e8143718e58 - 'calico/kube-controllers:v3.15.0': - sha1: f8921f5d67ee7db1c619aa9fdb74114569684ceb + 'calico/kube-controllers:v3.15.0': + sha1: f8921f5d67ee7db1c619aa9fdb74114569684ceb - 'calico/node:v3.15.0': - sha1: b15308e1aa8b9c56253c142e4361e47125bb4ac5 + 'calico/node:v3.15.0': + sha1: b15308e1aa8b9c56253c142e4361e47125bb4ac5 - 'calico/pod2daemon-flexvol:v3.15.0': - sha1: dd1a6525bde05937a28e3d9176b826162ae489af + 'calico/pod2daemon-flexvol:v3.15.0': + sha1: dd1a6525bde05937a28e3d9176b826162ae489af - # v1.19.15 - 'k8s.gcr.io/kube-apiserver:v1.19.15': - sha1: e01c8d778e4e693a0ea09cdbbe041a65cf070c6f + # v1.19.15 + 'k8s.gcr.io/kube-apiserver:v1.19.15': + sha1: e01c8d778e4e693a0ea09cdbbe041a65cf070c6f - 'k8s.gcr.io/kube-controller-manager:v1.19.15': - sha1: d1f5cc6a861b2259861fb78b2b83e9a07b788e31 + 'k8s.gcr.io/kube-controller-manager:v1.19.15': + sha1: d1f5cc6a861b2259861fb78b2b83e9a07b788e31 - 'k8s.gcr.io/kube-scheduler:v1.19.15': - sha1: b07fdd17205bc071ab108851d245689642244f92 + 'k8s.gcr.io/kube-scheduler:v1.19.15': + sha1: b07fdd17205bc071ab108851d245689642244f92 - 'k8s.gcr.io/kube-proxy:v1.19.15': - sha1: 9e2e7a8d40840bbade3a1f2dc743b9226491b6c2 + 'k8s.gcr.io/kube-proxy:v1.19.15': + sha1: 9e2e7a8d40840bbade3a1f2dc743b9226491b6c2 - # v1.20.12 - 'k8s.gcr.io/kube-apiserver:v1.20.12': - sha1: bbb037b9452db326aaf09988cee080940f3c418a + # v1.20.12 + 'k8s.gcr.io/kube-apiserver:v1.20.12': + sha1: bbb037b9452db326aaf09988cee080940f3c418a - 'k8s.gcr.io/kube-controller-manager:v1.20.12': - sha1: 4a902578a0c548edec93e0f4afea8b601fa54b93 + 'k8s.gcr.io/kube-controller-manager:v1.20.12': + sha1: 4a902578a0c548edec93e0f4afea8b601fa54b93 - 'k8s.gcr.io/kube-scheduler:v1.20.12': - sha1: ed5ceb21d0f5bc350db69550fb7feac7a6f1e50b + 'k8s.gcr.io/kube-scheduler:v1.20.12': + sha1: ed5ceb21d0f5bc350db69550fb7feac7a6f1e50b - 'k8s.gcr.io/kube-proxy:v1.20.12': - sha1: f937aba709f52be88360361230840e7bca756b2e + 'k8s.gcr.io/kube-proxy:v1.20.12': + sha1: f937aba709f52be88360361230840e7bca756b2e - 'k8s.gcr.io/coredns:1.7.0': - sha1: 5aa15f4cb942885879955b98a0a824833d9f66eb + 'k8s.gcr.io/coredns:1.7.0': + sha1: 5aa15f4cb942885879955b98a0a824833d9f66eb - 'k8s.gcr.io/pause:3.2': - sha1: ae4799e1a1ec9cd0dda8ab643b6e50c9fe505fef + 'k8s.gcr.io/pause:3.2': + sha1: ae4799e1a1ec9cd0dda8ab643b6e50c9fe505fef - # v1.21.7 - 'k8s.gcr.io/kube-apiserver:v1.21.7': - sha1: edb26859b3485808716982deccd90ca420828649 + # v1.21.7 + 'k8s.gcr.io/kube-apiserver:v1.21.7': + sha1: edb26859b3485808716982deccd90ca420828649 - 'k8s.gcr.io/kube-controller-manager:v1.21.7': - sha1: 9abf1841da5b113b377c1471880198259ec2d246 + 'k8s.gcr.io/kube-controller-manager:v1.21.7': + sha1: 9abf1841da5b113b377c1471880198259ec2d246 - 'k8s.gcr.io/kube-scheduler:v1.21.7': - sha1: 996d25351afc96a10e9008c04418db07a99c76b7 + 'k8s.gcr.io/kube-scheduler:v1.21.7': + sha1: 996d25351afc96a10e9008c04418db07a99c76b7 - 'k8s.gcr.io/kube-proxy:v1.21.7': - sha1: 450af22a892ffef276d4d58332b7817a1dde34e7 + 'k8s.gcr.io/kube-proxy:v1.21.7': + sha1: 450af22a892ffef276d4d58332b7817a1dde34e7 - 'k8s.gcr.io/coredns/coredns:v1.8.0': - sha1: 03114a98137e7cc2dcf4983b919e6b93ac8d1189 + 'k8s.gcr.io/coredns/coredns:v1.8.0': + sha1: 03114a98137e7cc2dcf4983b919e6b93ac8d1189 - 'k8s.gcr.io/etcd:3.4.13-0': - sha1: d37a2efafcc4aa86e6dc497e87e80b5d7f326115 + 'k8s.gcr.io/etcd:3.4.13-0': + sha1: d37a2efafcc4aa86e6dc497e87e80b5d7f326115 - 'k8s.gcr.io/pause:3.4.1': - sha1: 7f57ae28d733f99c0aab8f4e27d4b0c034cd0c04 + 'k8s.gcr.io/pause:3.4.1': + sha1: 7f57ae28d733f99c0aab8f4e27d4b0c034cd0c04 - # v1.22.4 - 'k8s.gcr.io/kube-apiserver:v1.22.4': - sha1: 2bf4ddb2e1f1530cf55ebaf8e8d0c56ad378b9ec + # v1.22.4 + 'k8s.gcr.io/kube-apiserver:v1.22.4': + sha1: 2bf4ddb2e1f1530cf55ebaf8e8d0c56ad378b9ec - 'k8s.gcr.io/kube-controller-manager:v1.22.4': - sha1: 241924fa3dc4671fe6644402f7beb60028c02c71 + 'k8s.gcr.io/kube-controller-manager:v1.22.4': + sha1: 241924fa3dc4671fe6644402f7beb60028c02c71 - 'k8s.gcr.io/kube-scheduler:v1.22.4': - sha1: 373e2939072b03cf5b1e115820b7fb6b749b0ebb + 'k8s.gcr.io/kube-scheduler:v1.22.4': + sha1: 373e2939072b03cf5b1e115820b7fb6b749b0ebb - 'k8s.gcr.io/kube-proxy:v1.22.4': - sha1: fecfb88509a430c29267a99b83f60f4a7c333583 + 'k8s.gcr.io/kube-proxy:v1.22.4': + sha1: fecfb88509a430c29267a99b83f60f4a7c333583 - 'k8s.gcr.io/coredns/coredns:v1.8.4': - sha1: 69c8e14ac3941fd5551ff22180be5f4ea2742d7f + 'k8s.gcr.io/coredns/coredns:v1.8.4': + sha1: 69c8e14ac3941fd5551ff22180be5f4ea2742d7f - 'k8s.gcr.io/etcd:3.5.0-0': - sha1: 9d9ee2df54a201dcc9c7a10ea763b9a5dce875f1 + 'k8s.gcr.io/etcd:3.5.0-0': + sha1: 9d9ee2df54a201dcc9c7a10ea763b9a5dce875f1 - 'k8s.gcr.io/pause:3.5': - sha1: bf3e3420df62f093f94c41d2b7a62b874dcbfc28 + 'k8s.gcr.io/pause:3.5': + sha1: bf3e3420df62f093f94c41d2b7a62b874dcbfc28 - 'quay.io/coreos/flannel:v0.14.0-amd64': - sha1: cff47465996a51de4632b53abf1fca873f147027 + 'quay.io/coreos/flannel:v0.14.0-amd64': + sha1: cff47465996a51de4632b53abf1fca873f147027 - 'quay.io/coreos/flannel:v0.14.0': - sha1: a487a36f7b31677e50e74b96b944f27fbce5ac13 + 'quay.io/coreos/flannel:v0.14.0': + sha1: a487a36f7b31677e50e74b96b944f27fbce5ac13 - 'calico/cni:v3.20.3': - sha1: 95e4cf79e92715b13e500a0efcfdb65590de1e04 + 'calico/cni:v3.20.3': + sha1: 95e4cf79e92715b13e500a0efcfdb65590de1e04 - 'calico/kube-controllers:v3.20.3': - sha1: 5769bae60830abcb3c5d97eb86b8f9938a587b2d + 'calico/kube-controllers:v3.20.3': + sha1: 5769bae60830abcb3c5d97eb86b8f9938a587b2d - 'calico/node:v3.20.3': - sha1: cc3c8727ad30b4850e8d0042681342a4f2351eff + 'calico/node:v3.20.3': + sha1: cc3c8727ad30b4850e8d0042681342a4f2351eff - 'calico/pod2daemon-flexvol:v3.20.3': - sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e + 'calico/pod2daemon-flexvol:v3.20.3': + sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e - # --- Rook --- - 'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0': - sha1: f076bd75359c6449b965c48eb8bad96c6d40790d + 'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0': + sha1: f076bd75359c6449b965c48eb8bad96c6d40790d - 'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0': - sha1: 129eb73c8e118e5049fee3d273b2d477c547e080 + 'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0': + sha1: 129eb73c8e118e5049fee3d273b2d477c547e080 - 'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0': - sha1: 2b45e5a3432cb89f3aec59584c1fa92c069e7a38 + 'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0': + sha1: 2b45e5a3432cb89f3aec59584c1fa92c069e7a38 - 'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0': - sha1: ce5c57454254c195762c1f58e1d902d7e81ea669 + 'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0': + sha1: ce5c57454254c195762c1f58e1d902d7e81ea669 - 'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1': - sha1: be1cf43617eea007629c0eb99149a99b6498f889 + 'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1': + sha1: be1cf43617eea007629c0eb99149a99b6498f889 - 'quay.io/ceph/ceph:v16.2.7': - sha1: fe9b7802c67e19111f83ffe4754ab62df66fd417 - allow_mismatch: true + 'quay.io/ceph/ceph:v16.2.7': + sha1: fe9b7802c67e19111f83ffe4754ab62df66fd417 + allow_mismatch: true - 'quay.io/cephcsi/cephcsi:v3.5.1': - sha1: 51dee9ea8ad76fb95ebd16f951e8ffaaaba95eb6 + 'quay.io/cephcsi/cephcsi:v3.5.1': + sha1: 51dee9ea8ad76fb95ebd16f951e8ffaaaba95eb6 - 'quay.io/csiaddons/k8s-sidecar:v0.2.1': - sha1: f0fd757436ac5075910c460c1991ff67c4774d09 + 'quay.io/csiaddons/k8s-sidecar:v0.2.1': + sha1: f0fd757436ac5075910c460c1991ff67c4774d09 - 'quay.io/csiaddons/volumereplication-operator:v0.3.0': - sha1: d3cd17f14fcbf09fc6c8c2c5c0419f098f87a70f + 'quay.io/csiaddons/volumereplication-operator:v0.3.0': + sha1: d3cd17f14fcbf09fc6c8c2c5c0419f098f87a70f - 'rook/ceph:v1.8.8': - sha1: f34039b17b18f5a855b096d48ff787b4013615e4 + rook: + 'rook/ceph:v1.8.8': + sha1: f34039b17b18f5a855b096d48ff787b4013615e4 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_cache.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_cache.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_cache.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_cache.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_key.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_key.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/apt_key.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_key.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf.py similarity index 93% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf.py index 1196237b74..b4db5124f9 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf.py @@ -24,11 +24,11 @@ class Dnf(DnfBase): Interface for `dnf` """ - def update(self, package: str = None, - disablerepo: str = None, - enablerepo: str = None, + def update(self, package: str = '', + disablerepo: str = '', + enablerepo: str = '', ignore_already_installed_error: bool = False, - releasever: str = None, + releasever: str = '', assume_yes: bool = True): """ Interface for `dnf update` @@ -46,16 +46,16 @@ def update(self, package: str = None, if assume_yes: update_parameters.append('-y') - if package is not None: + if package: update_parameters.append(package) - if disablerepo is not None: + if disablerepo: update_parameters.append(f'--disablerepo={disablerepo}') - if enablerepo is not None: + if enablerepo: update_parameters.append(f'--enablerepo={enablerepo}') - if releasever is not None: + if releasever: update_parameters.append(f'--releasever={releasever}') proc = self.run(update_parameters) @@ -87,7 +87,7 @@ def install(self, package: str, proc = self.run(['install', no_ask, package], accept_nonzero_returncode=True) if proc.returncode != 0: - if not 'does not update' in proc.stdout: # trying to reinstall package with url + if 'does not update' not in proc.stdout: # trying to reinstall package with url raise CriticalError(f'dnf install failed for `{package}`, reason `{proc.stdout}`') if 'error' in proc.stdout: diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_config_manager.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_config_manager.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_config_manager.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_config_manager.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_download.py similarity index 96% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_download.py index 5b46f00e19..c6240b49c1 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_download.py @@ -1,7 +1,7 @@ from pathlib import Path from typing import List -from src.command.dnf import DnfBase +from src.command.redhat.dnf import DnfBase from src.error import CriticalError diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_repoquery.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/dnf_repoquery.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/rpm.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/rpm.py similarity index 100% rename from ansible/playbooks/roles/repository/files/download-requirements/src/command/rpm.py rename to ansible/playbooks/roles/repository/files/download-requirements/src/command/redhat/rpm.py diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py index 5aba587f13..7c2488c175 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/toolchain.py @@ -1,17 +1,17 @@ import logging from typing import Dict -from src.command.apt import Apt -from src.command.apt_cache import AptCache -from src.command.apt_key import AptKey from src.command.crane import Crane -from src.command.dnf_repoquery import DnfRepoquery -from src.command.rpm import Rpm +from src.command.debian.apt import Apt +from src.command.debian.apt_cache import AptCache +from src.command.debian.apt_key import AptKey +from src.command.redhat.dnf import Dnf +from src.command.redhat.dnf_config_manager import DnfConfigManager +from src.command.redhat.dnf_download import DnfDownload +from src.command.redhat.dnf_repoquery import DnfRepoquery +from src.command.redhat.rpm import Rpm from src.command.tar import Tar from src.command.wget import Wget -from src.command.dnf import Dnf -from src.command.dnf_config_manager import DnfConfigManager -from src.command.dnf_download import DnfDownload from src.config.os_type import OSFamily diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py index b4c9d53b43..c4f29ba97c 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/config.py @@ -4,10 +4,10 @@ from itertools import chain from os import uname from pathlib import Path -from typing import List +from typing import Any, Dict, List from src.config.os_type import OSArch, OSConfig, OSType, SUPPORTED_OS_TYPES -from src.error import CriticalError +from src.error import CriticalError, OldManifestVersion class Config: @@ -17,23 +17,28 @@ def __init__(self, argv: List[str]): self.dest_files: Path self.dest_grafana_dashboards: Path self.dest_images: Path + self.dest_manifest: Path self.dest_packages: Path self.distro_subdir: Path self.is_log_file_enabled: bool self.log_file: Path + self.log_level: int self.os_arch: OSArch self.os_type: OSType self.pyyaml_installed: bool = False self.repo_path: Path self.repos_backup_file: Path self.reqs_path: Path - self.rerun: bool + self.rerun: bool = False self.retries: int self.script_path: Path + self.verbose_mode: bool self.was_backup_created: bool = False self.__add_args(argv) + self.__LINE_SIZE: int = 50 # used in printing + if not self.rerun: self.__log_info_summary() @@ -43,8 +48,7 @@ def __log_info_summary(self): """ lines: List[str] = ['Info summary:'] - LINE_SIZE: int = 50 - lines.append('-' * LINE_SIZE) + lines.append('-' * self.__LINE_SIZE) lines.append(f'OS Arch: {self.os_arch.value}') lines.append(f'OS Type: {self.os_type.os_name}') @@ -56,12 +60,16 @@ def __log_info_summary(self): lines.append(f'- packages: {str(self.dest_packages)}') lines.append(f'Repos backup file: {str(self.repos_backup_file)}') + if self.dest_manifest: + lines.append(f'Manifest used: {str(self.dest_manifest.absolute())}') + if self.is_log_file_enabled: lines.append(f'Log file location: {str(self.log_file.absolute())}') + lines.append(f'Verbose mode: {self.verbose_mode}') lines.append(f'Retries count: {self.retries}') - lines.append('-' * LINE_SIZE) + lines.append('-' * self.__LINE_SIZE) logging.info('\n'.join(lines)) @@ -93,6 +101,12 @@ def __create_parser(self) -> ArgumentParser: parser.add_argument('--no-logfile', action='store_true', dest='no_logfile', help='no logfile will be created') + parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', + help='more verbose output will be provided') + + parser.add_argument('--manifest', '-m', metavar='MANIFEST_PATH', type=Path, action='store', dest='manifest', + help='manifest file generated by epicli') + # offline mode rerun options: parser.add_argument('--rerun', action='store_true', dest='rerun', default=False, help=SUPPRESS) @@ -144,18 +158,19 @@ def __setup_logger(self, log_level: str, log_file: Path, no_logfile: bool): 'info': logging.INFO, 'debug': logging.DEBUG } + self.log_level = log_levels[log_level.lower()] log_format = '%(asctime)s [%(levelname)s]: %(message)s' # add stdout logger: - logging.basicConfig(stream=sys.stdout, level=log_levels[log_level.lower()], + logging.basicConfig(stream=sys.stdout, level=self.log_level, format=log_format) # add log file: if not no_logfile: root_logger = logging.getLogger() file_handler = logging.FileHandler(log_file) - file_handler.setLevel(log_levels[log_level.lower()]) + file_handler.setLevel(self.log_level) file_handler.setFormatter(logging.Formatter(fmt=log_format)) root_logger.addHandler(file_handler) @@ -195,7 +210,132 @@ def __add_args(self, argv: List[str]): self.repos_backup_file = Path(args['repos_backup_file']) self.retries = args['retries'] self.is_log_file_enabled = False if args['no_logfile'] else True + self.dest_manifest = args['manifest'] or None + self.verbose_mode = True if self.log_level == logging.DEBUG else args['verbose'] # offline mode self.rerun = args['rerun'] self.pyyaml_installed = args['pyyaml_installed'] + + def __print_parsed_manifest_data(self, requirements: Dict[str, Any], manifest: Dict[str, Any]): + lines: List[str] = ['Manifest summary:'] + + lines.append('-' * self.__LINE_SIZE) + + lines.append('Components requested:') + for component in manifest['requested-components']: + lines.append(f'- {component}') + + lines.append('') + + lines.append('Features requested:') + for feature in manifest['requested-features']: + lines.append(f'- {feature}') + + for reqs in [('files', 'Files'), + ('grafana-dashboards', 'Dashboards')]: + reqs_to_download = sorted(requirements[reqs[0]]) + if reqs_to_download: + lines.append('') + lines.append(f'{reqs[1]} to download:') + for req_to_download in reqs_to_download: + lines.append(f'- {req_to_download}') + + images = requirements['images'] + images_to_print: List[str] = [] + for image_category in images: + for image in images[image_category]: + images_to_print.append(image) + + if images_to_print: + lines.append('') + lines.append('Images to download:') + for image in sorted(images_to_print): + lines.append(f'- {image}') + + lines.append('-' * self.__LINE_SIZE) + + logging.info('\n'.join(lines)) + + def __filter_files(self, requirements: Dict[str, Any], + manifest: Dict[str, Any], + is_k8s_as_cloud_service: bool): + """ + See :func:`~config.Config.__filter_manifest` + """ + files = requirements['files'] + files_to_exclude: List[str] = [] + for file in files: + deps = files[file]['deps'] + if all(dep not in manifest['requested-features'] for dep in deps) and deps != 'default': + files_to_exclude.append(file) + + if is_k8s_as_cloud_service: + files_to_exclude = [file for file in files if 'kubernetes-master' not in files[file]['deps']] + + if files_to_exclude: + requirements['files'] = {url: data for url, data in files.items() if url not in files_to_exclude} + + def __filter_images(self, requirements: Dict[str, Any], manifest: Dict[str, Any]): + """ + See :func:`~config.Config.__filter_manifest` + """ + # prepare image groups: + images = requirements['images'] + images_to_download: Dict[str, Dict] = {} + for image_group in images: + images_to_download[image_group] = {} + + if len(manifest['requested-images']): # if image-registry document used: + for image_group in images: + for image, data in images[image_group].items(): + if image in manifest['requested-images']: + images_to_download[image_group][image] = data + else: # otherwise check features used: + for image_group in images: + if image_group in manifest['requested-features']: + for image, data in images[image_group].items(): + images_to_download[image_group][image] = data + + if images_to_download: + requirements['images'] = images_to_download + + def __filter_manifest(self, requirements: Dict[str, Any], + manifest: Dict[str, Any], + is_k8s_as_cloud_service: bool): + """ + Filter entries in the `requirements` based on the parsed `manifest` documents. + + :param requirements: parsed requirements which will be filtered based on the `manifest` content + :param manifest: parsed documents which will be used to filter `requirements` + :param is_k8s_as_cloud_service: whether to download kubernetes files if service is not present + """ + if 'grafana' not in manifest['requested-features']: + requirements['grafana-dashboards'] = [] + + self.__filter_files(requirements, manifest, is_k8s_as_cloud_service) + self.__filter_images(requirements, manifest) + + def read_manifest(self, requirements: Dict[str, Any]): + """ + Construct ManifestReader and parse only required data. + Not needed entries will be removed from the `requirements` + + :param requirements: parsed requirements which will be filtered based on the manifest output + """ + if not self.dest_manifest: + return + + # Needs to be imported here as the libyaml might be missing on the OS, + # this could cause crash on config.py import. + from src.config.manifest_reader import ManifestReader + + mreader = ManifestReader(self.dest_manifest, self.os_arch) + try: + manifest = mreader.parse_manifest() + self.__filter_manifest(requirements, manifest, mreader.is_k8s_as_cloud_service) + + if self.verbose_mode: + self.__print_parsed_manifest_data(requirements, manifest) + except OldManifestVersion: + pass # old manifest used, cannot optimize download time diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py new file mode 100644 index 0000000000..a0d8af6e6a --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/manifest_reader.py @@ -0,0 +1,117 @@ +from pathlib import Path +from typing import Any, Callable, Dict, List, Set + +import yaml + +from src.config.os_type import OSArch +from src.config.version import Version +from src.error import CriticalError, OldManifestVersion + + +def load_yaml_file_all(filename: Path) -> List[Any]: + try: + with open(filename, encoding="utf-8") as req_handler: + return list(yaml.safe_load_all(req_handler)) + except yaml.YAMLError as yaml_err: + raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err + except Exception as err: + raise CriticalError(f'Failed loading: `{filename}`') from err + + +def load_yaml_file(filename: Path) -> Any: + return load_yaml_file_all(filename)[0] + + +class ManifestReader: + """ + Load the manifest file and call defined parser methods to process required documents. + Main running method is :func:`~manifest_reader.ManifestReader.parse_manifest` which returns formatted manifest output. + """ + + def __init__(self, dest_manifest: Path, arch: OSArch): + self.__dest_manifest = dest_manifest + self.__os_arch: str = arch.value + + self.__k8s_as_cloud_service: bool = False + + self.__requested_components: Set = set() + self.__requested_features: Set = set() + self.__requested_images: Set = set() + + @property + def is_k8s_as_cloud_service(self) -> bool: + return self.__k8s_as_cloud_service + + def __parse_cluster_doc(self, cluster_doc: Dict): + """ + Parse `epiphany-cluster` document and extract only used components. + + :param cluster_doc: handler to a `epiphany-cluster` document + :raises: + :class:`OldManifestVersion`: can be raised when old manifest version used + """ + if Version(cluster_doc['version']) < Version('2.0.1'): + raise OldManifestVersion(cluster_doc['version']) + + try: + self.__k8s_as_cloud_service = cluster_doc['specification']['cloud']['k8s_as_cloud_service'] + except KeyError: + self.__k8s_as_cloud_service = False + + components = cluster_doc['specification']['components'] + for component in components: + if components[component]['count'] > 0: + self.__requested_components.add(component) + + def __parse_feature_mappings_doc(self, feature_mappings_doc: Dict): + """ + Parse `configuration/feature-mappings` document and extract only used features (based on `epiphany-cluster` doc). + + :param feature_mappings_doc: handler to a `configuration/feature-mappings` document + """ + mappings = feature_mappings_doc['specification']['mappings'] + for mapping in mappings.keys() & self.__requested_components: + for feature in mappings[mapping]: + self.__requested_features.add(feature) + + def __parse_image_registry_doc(self, image_registry_doc: Dict): + """ + Parse `configuration/image-registry` document and extract only used images. + + :param image_registry_doc: handler to a `configuration/image-registry` document + """ + self.__requested_images.add(image_registry_doc['specification']['registry_image']['name']) + + target_arch_images = image_registry_doc['specification']['images_to_load'][self.__os_arch] + for target_images in target_arch_images: + features = target_arch_images[target_images] + for feature in features: + for image in features[feature]: + self.__requested_images.add(image['name']) + + def parse_manifest(self) -> Dict[str, Any]: + """ + Load the manifest file, call parsers on required docs and return formatted output. + """ + required_docs: Set[str] = {'epiphany-cluster', 'configuration/feature-mappings'} + parse_doc: Dict[str, Callable] = { + 'epiphany-cluster': self.__parse_cluster_doc, + 'configuration/feature-mappings': self.__parse_feature_mappings_doc, + 'configuration/image-registry': self.__parse_image_registry_doc + } + + parsed_docs: Set[str] = set() + for manifest_doc in load_yaml_file_all(self.__dest_manifest): + try: + kind: str = manifest_doc['kind'] + parse_doc[kind](manifest_doc) + parsed_docs.add(kind) + except KeyError: + pass + + if len(parsed_docs) < len(required_docs): + raise CriticalError(f'ManifestReader - could not find document(s): {parsed_docs ^ required_docs}') + + return {'requested-components': sorted(list(self.__requested_components)), + 'requested-features': sorted(list(self.__requested_features)), + 'requested-images': sorted(list(self.__requested_images))} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/config/version.py b/ansible/playbooks/roles/repository/files/download-requirements/src/config/version.py new file mode 100644 index 0000000000..fd0de4a2ce --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/config/version.py @@ -0,0 +1,24 @@ +class Version: + """ + Type used in comparing epiphany version. + """ + + def __init__(self, ver: str): + major, minor, patch = ver.split('.') + self.major: int = int(major) + self.minor: int = int(minor) + self.patch: int = int(''.join(filter(lambda char: char.isdigit(), patch))) # handle `1dev`, `2rc`, etc. + + def __lt__(self, rhs): + if self.major < rhs.major: + return True + + if self.major == rhs.major: + if self.minor < rhs.minor: + return True + + if self.minor == rhs.minor: + if self.patch < rhs.patch: + return True + + return False diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/error.py b/ansible/playbooks/roles/repository/files/download-requirements/src/error.py index 0e7b9482f7..6029e014b7 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/error.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/error.py @@ -1,7 +1,13 @@ import logging -class DownloadRequirementsError(Exception): +class DownloadRequirementsException(Exception): + """ + Base class for all exceptions raised during the script runtime. + """ + + +class DownloadRequirementsError(DownloadRequirementsException): """ Base class for all non standard errors raised during a script run. """ @@ -10,6 +16,15 @@ def __init__(self, msg: str): logging.error(msg) +class DownloadRequirementsWarning(DownloadRequirementsException): + """ + Base class for all non critical issues raised during a script run. + """ + def __init__(self, msg: str): + super().__init__() + logging.warning(msg) + + class CriticalError(DownloadRequirementsError): """ Raised when there was an error that could not be fixed by @@ -35,4 +50,12 @@ class ChecksumMismatch(DownloadRequirementsError): """ def __init__(self, msg: str): super().__init__(f'{msg} - download failed due to checksum mismatch, ' - 'WARNING someone might have replaced the file') + 'WARNING someone might have replaced the file.') + + +class OldManifestVersion(DownloadRequirementsWarning): + """ + Raised when old manifest version used + """ + def __init__(self, version: str): + super().__init__(f'Old manifest version used: `{version}`, no optimization will be performed.') diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py index 980841243b..535b652371 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/base_mode.py @@ -4,25 +4,14 @@ from pathlib import Path from typing import Any, Dict -import yaml - from src.command.toolchain import Toolchain, TOOLCHAINS from src.config.config import Config, OSArch +from src.config.manifest_reader import load_yaml_file from src.crypt import SHA_ALGORITHMS from src.downloader import Downloader from src.error import CriticalError, ChecksumMismatch -def load_yaml_file(filename: Path) -> Any: - try: - with open(filename, encoding="utf-8") as req_handler: - return yaml.safe_load(req_handler) - except yaml.YAMLError as yaml_err: - raise CriticalError(f'Failed loading: `{yaml_err}`') from yaml_err - except Exception as err: - raise CriticalError(f'Failed loading: `{filename}`') from err - - class BaseMode: """ An abstract class for running specific operations on target OS. @@ -35,6 +24,7 @@ def __init__(self, config: Config): self._repositories: Dict[str, Dict] = self.__parse_repositories() self._requirements: Dict[str, Any] = self.__parse_requirements() self._tools: Toolchain = TOOLCHAINS[self._cfg.os_type.os_family](self._cfg.retries) + self._cfg.read_manifest(self._requirements) def __parse_repositories(self) -> Dict[str, Dict]: """ @@ -191,12 +181,19 @@ def _download_images(self): Download images under `self._requirements['images']` using Crane. """ platform: str = 'linux/amd64' if self._cfg.os_arch == OSArch.X86_64 else 'linux/arm64' - downloader: Downloader = Downloader(self._requirements['images'], + images = self._requirements['images'] + + images_to_download: Dict[str, Dict] = {} + for image_group in images: # kubernetes-master, rabbitmq, etc. + for image, data in images[image_group].items(): + images_to_download[image] = data + + downloader: Downloader = Downloader(images_to_download, 'sha1', self._tools.crane.pull, {'platform': platform}) - for image in self._requirements['images']: + for image in images_to_download: url, version = image.split(':') filename = Path(f'{url.split("/")[-1]}-{version}.tar') # format: image_version.tar @@ -245,10 +242,12 @@ def run(self): """ # add required directories self._cfg.dest_files.mkdir(exist_ok=True, parents=True) - self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True) self._cfg.dest_images.mkdir(exist_ok=True, parents=True) self._cfg.dest_packages.mkdir(exist_ok=True, parents=True) + if self._requirements['grafana-dashboards']: + self._cfg.dest_grafana_dashboards.mkdir(exist_ok=True, parents=True) + # provides tar which is required for backup logging.info('Installing base packages...') self._install_base_packages() @@ -275,9 +274,10 @@ def run(self): self.__download_files(self._requirements['files'], self._cfg.dest_files) logging.info('Done downloading files.') - logging.info('Downloading grafana dashboards...') - self.__download_grafana_dashboards() - logging.info('Done downloading grafana dashboards.') + if self._requirements['grafana-dashboards']: + logging.info('Downloading grafana dashboards...') + self.__download_grafana_dashboards() + logging.info('Done downloading grafana dashboards.') logging.info('Downloading Crane...') self.__download_crane() diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py similarity index 97% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py index fa91bb4e44..086a9aeb32 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt import Apt +from src.command.debian.apt import Apt def test_interface_update(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py similarity index 93% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py index 4a7008f476..359eb40238 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_cache.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt_cache import AptCache +from src.command.debian.apt_cache import AptCache def test_interface_get_package_dependencies(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py similarity index 88% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py index 980398399d..d247466d68 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_apt_key.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_key.py @@ -2,7 +2,7 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.apt_key import AptKey +from src.command.debian.apt_key import AptKey def test_interface_add(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py similarity index 97% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py index fc94fc92e8..78c1aa6d87 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf.py @@ -1,7 +1,7 @@ -from src.command.dnf import Dnf - from tests.mocks.command_run_mock import CommandRunMock +from src.command.redhat.dnf import Dnf + def test_interface_update(mocker): ''' Check argument construction for `dnf update` ''' diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_base.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_base.py similarity index 86% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_base.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_base.py index 0f2c377c1a..a40a92a167 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_base.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_base.py @@ -1,4 +1,4 @@ -from src.command.dnf import DnfBase +from src.command.redhat.dnf import DnfBase def test_filter_non_critical_errors(): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py similarity index 93% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py index 9835bebef9..ae9ab2dc47 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_config_manager.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_config_manager.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_config_manager import DnfConfigManager +from src.command.redhat.dnf_config_manager import DnfConfigManager def test_interface_add_repo(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py similarity index 94% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py index 4b689d6ffb..911aeee576 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_download.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_download.py @@ -2,7 +2,7 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_download import DnfDownload +from src.command.redhat.dnf_download import DnfDownload def test_interface_download_packages(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py similarity index 96% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py index 158509576d..44a9b94dc9 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_dnf_repoquery.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.dnf_repoquery import DnfRepoquery +from src.command.redhat.dnf_repoquery import DnfRepoquery def test_interface_query(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py similarity index 96% rename from ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py rename to ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py index 57baf41eb7..1423889d44 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_rpm.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/redhat/test_rpm.py @@ -1,6 +1,6 @@ from tests.mocks.command_run_mock import CommandRunMock -from src.command.rpm import Rpm +from src.command.redhat.rpm import Rpm def test_interface_is_package_installed(mocker): diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py new file mode 100644 index 0000000000..816f3799f3 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_config.py @@ -0,0 +1,75 @@ +from pathlib import Path +import logging + +import pytest +import yaml + +from src.config.config import Config +from tests.data.config import ( + DASHBOARD_REQUIREMENTS, + EXPECTED_VERBOSE_DASHBOARD_OUTPUT, + EXPECTED_VERBOSE_FILE_OUTPUT, + EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT, + EXPECTED_VERBOSE_IMAGE_OUTPUT, + EXPECTED_VERBOSE_K8S_AS_CLOUD_SERVICE_OUTPUT, + EXPECTED_VERBOSE_OUTPUT, + FILE_REQUIREMENTS, + IMAGE_REQUIREMENTS +) +from tests.data.manifest_reader import ( + INPUT_MANIFEST_FEATURE_MAPPINGS, + INPUT_MANIFEST_IMAGES_NO_DOCUMENT, + INPUT_MANIFEST_WITH_DASHBOARDS, + INPUT_MANIFEST_WITH_IMAGES, + INPUT_MANIFEST_WITH_K8S_AS_CLOUD_SERVICE +) +from src.config.os_type import OSArch + + +@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, REQUIREMENTS', + [ + (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_FILE_OUTPUT, FILE_REQUIREMENTS), + (INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_VERBOSE_OUTPUT, DASHBOARD_REQUIREMENTS), + (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_VERBOSE_DASHBOARD_OUTPUT, DASHBOARD_REQUIREMENTS), + (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_VERBOSE_IMAGE_OUTPUT, IMAGE_REQUIREMENTS), + (INPUT_MANIFEST_IMAGES_NO_DOCUMENT, EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT, IMAGE_REQUIREMENTS), + (INPUT_MANIFEST_WITH_K8S_AS_CLOUD_SERVICE, EXPECTED_VERBOSE_K8S_AS_CLOUD_SERVICE_OUTPUT, FILE_REQUIREMENTS) + ]) +def test_manifest_verbose_output(INPUT_DOC: str, + EXPECTED_OUTPUT_DOC: str, + REQUIREMENTS: str, + mocker, caplog): + """ + Check output produced when running download-requirements script with the `-v|--verbose` flag and with provided `-m|--manifest` + + :param INPUT_DOC: yaml doc which will be parsed by the ManifestReader + :param EXPECTED_OUTPUT_DOC: expected output to be printed by the `Config` class, then tested against the parsed `INPUT_DOC` + :param REQUIREMENTS: yaml doc containing requirements passed to `Config`'s read_manifest() + """ + MANIFEST = { + 'files': '', + 'grafana-dashboards': '', + 'images': '' + } + + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) + caplog.set_level(logging.INFO) + + # mock Config's init methods: + Config._Config__add_args = lambda *args: None + Config._Config__log_info_summary = lambda *args: None + + config = Config([]) + + # mock required config data: + config.dest_manifest = Path('/some/path') + config.os_arch = OSArch.X86_64 + config.verbose_mode = True + + req_key, doc = tuple(yaml.safe_load(REQUIREMENTS).items())[0] + MANIFEST[req_key] = doc + config.read_manifest(MANIFEST) + + log_output = f'\n{"".join(caplog.messages)}\n' + + assert log_output == EXPECTED_OUTPUT_DOC diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py new file mode 100644 index 0000000000..933e5f99db --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_manifest_reader.py @@ -0,0 +1,26 @@ +from pathlib import Path + +import pytest +import yaml + +from src.config.manifest_reader import ManifestReader +from src.config.os_type import OSArch +from tests.data.manifest_reader import (EXPECTED_FEATURE_MAPPINGS, + EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, + EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64, + EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64, + INPUT_MANIFEST_FEATURE_MAPPINGS, + INPUT_MANIFEST_WITH_DASHBOARDS, + INPUT_MANIFEST_WITH_IMAGES) + +@pytest.mark.parametrize('INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH', + [(INPUT_MANIFEST_FEATURE_MAPPINGS, EXPECTED_FEATURE_MAPPINGS, OSArch.X86_64), + (INPUT_MANIFEST_WITH_DASHBOARDS, EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS, OSArch.X86_64), + (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64, OSArch.X86_64), + (INPUT_MANIFEST_WITH_IMAGES, EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64, OSArch.ARM64)]) +def test_parse_manifest(INPUT_DOC, EXPECTED_OUTPUT_DOC, OS_ARCH, mocker): + ''' Check manifest file parsing ''' + mocker.patch('src.config.manifest_reader.load_yaml_file_all', return_value=yaml.safe_load_all(INPUT_DOC)) + + mreader = ManifestReader(Path('/some/path'), OS_ARCH) + assert mreader.parse_manifest() == EXPECTED_OUTPUT_DOC diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_version.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_version.py new file mode 100644 index 0000000000..0e9e22bd09 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/config/test_version.py @@ -0,0 +1,17 @@ +from src.config.version import Version + + +def test_version_major(): + assert Version('1.2.4') < Version('2.3.0dev') + + +def test_version_minor(): + assert Version('1.2.4') < Version('1.3.0dev') + + +def test_version_patch(): + assert Version('1.2.4') < Version('1.2.5dev') + + +def test_version_not(): + assert not (Version('1.2.4') < Version('1.1.0')) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py new file mode 100644 index 0000000000..d2d3492027 --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/config.py @@ -0,0 +1,475 @@ +FILE_REQUIREMENTS = """ +files: + # --- Exporters --- + 'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz': + sha256: ffda682e82daede726da8719257a088f8e23dcaa4e2ac8b2b2748a129aea85f0 + deps: [kafka-exporter] + + 'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar': + sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e + deps: [kafka] + + 'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz': + sha256: 68f3802c2dd3980667e4ba65ea2e1fb03f4a4ba026cca375f15a0390ff850949 + deps: [node-exporter] + + 'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-amd64.tar.gz': + sha256: 1d1a008c5e29673b404a9ce119b7516fa59974aeda2f47d4a0446d102abce8a1 + deps: [postgres-exporter] + + # --- Misc --- + 'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz': + sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98 + deps: [kafka] + + 'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz': + sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3 + deps: [zookeeper] + + 'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz': + sha256: 77793c4d9bb92be98f7525f8bc50cb8adb8c5de2e944d5500e90ab13918771fc + deps: [prometheus] + + 'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-amd64.tar.gz': + sha256: 7852dc11cfaa039577c1804fe6f082a07c5eb06be50babcffe29214aedf318b3 + deps: [prometheus] + + 'https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz': + sha256: 4c3fd562e64005786ac8f18e7334054a24da34ec04bbd769c206b03b8ed6e457 + deps: [helm] + + # --- Helm charts --- + 'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz': + sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f + deps: [kubernetes-master] + + 'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz': + sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79 + deps: [kubernetes-master] + + 'https://charts.rook.io/release/rook-ceph-v1.8.8.tgz': + sha256: f67e474dedffd4004f3a0b7b40112694a7f1c2b1a0048b03b3083d0a01e86b14 + deps: [kubernetes-master] + + 'https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz': + sha256: df4e1f2125af41fb84c72e4d12aa0cb859dddd4f37b3d5979981bd092040bd16 + deps: [kubernetes-master] + + # --- OpenSearch Bundle --- + 'https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-x64.tar.gz': + sha256: d40f2696623b6766aa235997e2847a6c661a226815d4ba173292a219754bd8a8 + deps: [logging, opensearch] + + 'https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-x64.tar.gz': + sha256: 14623798e61be6913e2a218d6ba3e308e5036359d7bda58482ad2f1340aa3c85 + deps: [opensearch-dashboards] + + 'https://github.com/opensearch-project/perftop/releases/download/1.2.0.0/opensearch-perf-top-1.2.0.0-linux-x64.zip': + sha256: e8f9683976001a8cf59a9f86da5caafa10b88643315f0af2baa93a9354d41e2b + deps: [logging, opensearch] +""" + + +DASHBOARD_REQUIREMENTS = """ +grafana-dashboards: + grafana_dashboard_7249: + url: 'https://grafana.com/api/dashboards/7249/revisions/1/download' + sha256: 41cc2794b1cc9fc537baf045fee12d086d23632b4c8b2e88985274bb9862e731 + grafana_dashboard_315: + url: 'https://grafana.com/api/dashboards/315/revisions/3/download' + sha256: ee46dd6e68a9950aa78e8c88ae5e565c8ebde6cbdbe08972a70f06c5486618fb + grafana_dashboard_11074: + url: 'https://grafana.com/api/dashboards/11074/revisions/9/download' + sha256: 151b23305da46eab84930e99175e1c07e375af73dbbb4b8f501ca25f5ac62785 + grafana_dashboard_405: + url: 'https://grafana.com/api/dashboards/405/revisions/8/download' + sha256: 97675027cbd5b7241e93a2b598654c4b466bc909eeb6358ba123d500094d913c + grafana_dashboard_455: + url: 'https://grafana.com/api/dashboards/455/revisions/2/download' + sha256: c66b91ab8d258b0dc005d3ee4dac3a5634a627c79cc8053875f76ab1e369a362 + grafana_dashboard_9628: + url: 'https://grafana.com/api/dashboards/9628/revisions/7/download' + sha256: c64cc38ad9ebd7af09551ee83e669a38f62a76e7c80929af5668a5852732b376 + grafana_dashboard_4279: + url: 'https://grafana.com/api/dashboards/4279/revisions/4/download' + sha256: 74d47be868da52c145240ab5586d91ace9e9218ca775af988f9d60e501907a25 + grafana_dashboard_1860: + url: 'https://grafana.com/api/dashboards/1860/revisions/23/download' + sha256: 225faab8bf35c1723af14d4c069882ccb92b455d1941c6b1cf3d95a1576c13d7 + grafana_dashboard_7589: + url: 'https://grafana.com/api/dashboards/7589/revisions/5/download' + sha256: cf020e14465626360418e8b5746818c80d77c0301422f3060879fddc099c2151 + grafana_dashboard_789: + url: 'https://grafana.com/api/dashboards/789/revisions/1/download' + sha256: 6a9b4bdc386062287af4f7d56781103a2e45a51813596a65f03c1ae1d4d3e919 + grafana_dashboard_179: + url: 'https://grafana.com/api/dashboards/179/revisions/7/download' + sha256: 8d67350ff74e715fb1463f2406f24a73377357d90344f8200dad9d1b2a8133c2 + grafana_dashboard_6663: + url: 'https://grafana.com/api/dashboards/6663/revisions/1/download' + sha256: d544d88069e1b793ff3d8f6970df641ad9a66217e69b629621e1ecbb2f06aa05 + grafana_dashboard_10991: + url: 'https://grafana.com/api/dashboards/10991/revisions/11/download' + sha256: 66340fa3256d432287cba75ab5177eb058c77afa7d521a75d58099f95b1bff50 +""" + +IMAGE_REQUIREMENTS = """ +images: + haproxy: + 'haproxy:2.2.2-alpine': + sha1: dff8993b065b7f7846adb553548bcdcfcd1b6e8e + + image-registry: + 'registry:2.8.0': + sha1: 89795c17099199c752d02ad8797c1d4565a08aff + allow_mismatch: true + + applications: + 'bitnami/pgpool:4.2.4': + sha1: 66741f3cf4a508bd1f80e2965b0086a4c0fc3580 + + 'bitnami/pgbouncer:1.16.0': + sha1: f2e37eecbf9aed44d5566f06dcc101c1ba9edff9 + + 'epiphanyplatform/keycloak:14.0.0': + sha1: b59d75a967cedd3a4cf5867eced2fb5dff52f60e + + 'rabbitmq:3.8.9': + sha1: c64408bf5bb522f47d5323652dd5e60560dcb5bc + + kubernetes-master: + 'kubernetesui/dashboard:v2.3.1': + sha1: 8c8a4ac7a643f9c5dd9e5d22876c434187312db8 + + 'kubernetesui/metrics-scraper:v1.0.7': + sha1: 5a0052e2afd3eef3ae638be21938b29b1d608ebe + + # K8s + # v1.18.6 + 'k8s.gcr.io/kube-apiserver:v1.18.6': + sha1: 164968226f4617abaa31e6108ed9034a1e302f4f + + 'k8s.gcr.io/kube-controller-manager:v1.18.6': + sha1: ebea3fecab9e5693d31438fa37dc4d02c6914d67 + + 'k8s.gcr.io/kube-scheduler:v1.18.6': + sha1: 183d29c4fdcfda7478d08240934fdb6845e2e3ec + + 'k8s.gcr.io/kube-proxy:v1.18.6': + sha1: 62da886e36efff0c03a16e19c1442a1c3040fbf1 + + 'k8s.gcr.io/coredns:1.6.7': + sha1: 76615ffabb22fd4fb3d562cb6ebcd243f8826e48 + + 'k8s.gcr.io/etcd:3.4.3-0': + sha1: 6ee82ddb1bbc7f1831c42046612b8bcfbb171b45 + + 'quay.io/coreos/flannel:v0.12.0-amd64': + sha1: 3516522e779373983992095e61eb6615edd50d1f + + 'quay.io/coreos/flannel:v0.12.0': + sha1: 2cb6ce8f1361886225526767c4a0422c039453c8 + + 'calico/cni:v3.15.0': + sha1: aa59f624c223bc398a42c7ba9e628e8143718e58 + + 'calico/kube-controllers:v3.15.0': + sha1: f8921f5d67ee7db1c619aa9fdb74114569684ceb + + 'calico/node:v3.15.0': + sha1: b15308e1aa8b9c56253c142e4361e47125bb4ac5 + + 'calico/pod2daemon-flexvol:v3.15.0': + sha1: dd1a6525bde05937a28e3d9176b826162ae489af + + # v1.19.15 + 'k8s.gcr.io/kube-apiserver:v1.19.15': + sha1: e01c8d778e4e693a0ea09cdbbe041a65cf070c6f + + 'k8s.gcr.io/kube-controller-manager:v1.19.15': + sha1: d1f5cc6a861b2259861fb78b2b83e9a07b788e31 + + 'k8s.gcr.io/kube-scheduler:v1.19.15': + sha1: b07fdd17205bc071ab108851d245689642244f92 + + 'k8s.gcr.io/kube-proxy:v1.19.15': + sha1: 9e2e7a8d40840bbade3a1f2dc743b9226491b6c2 + + # v1.20.12 + 'k8s.gcr.io/kube-apiserver:v1.20.12': + sha1: bbb037b9452db326aaf09988cee080940f3c418a + + 'k8s.gcr.io/kube-controller-manager:v1.20.12': + sha1: 4a902578a0c548edec93e0f4afea8b601fa54b93 + + 'k8s.gcr.io/kube-scheduler:v1.20.12': + sha1: ed5ceb21d0f5bc350db69550fb7feac7a6f1e50b + + 'k8s.gcr.io/kube-proxy:v1.20.12': + sha1: f937aba709f52be88360361230840e7bca756b2e + + 'k8s.gcr.io/coredns:1.7.0': + sha1: 5aa15f4cb942885879955b98a0a824833d9f66eb + + 'k8s.gcr.io/pause:3.2': + sha1: ae4799e1a1ec9cd0dda8ab643b6e50c9fe505fef + + # v1.21.7 + 'k8s.gcr.io/kube-apiserver:v1.21.7': + sha1: edb26859b3485808716982deccd90ca420828649 + + 'k8s.gcr.io/kube-controller-manager:v1.21.7': + sha1: 9abf1841da5b113b377c1471880198259ec2d246 + + 'k8s.gcr.io/kube-scheduler:v1.21.7': + sha1: 996d25351afc96a10e9008c04418db07a99c76b7 + + 'k8s.gcr.io/kube-proxy:v1.21.7': + sha1: 450af22a892ffef276d4d58332b7817a1dde34e7 + + 'k8s.gcr.io/coredns/coredns:v1.8.0': + sha1: 03114a98137e7cc2dcf4983b919e6b93ac8d1189 + + 'k8s.gcr.io/etcd:3.4.13-0': + sha1: d37a2efafcc4aa86e6dc497e87e80b5d7f326115 + + 'k8s.gcr.io/pause:3.4.1': + sha1: 7f57ae28d733f99c0aab8f4e27d4b0c034cd0c04 + + # v1.22.4 + 'k8s.gcr.io/kube-apiserver:v1.22.4': + sha1: 2bf4ddb2e1f1530cf55ebaf8e8d0c56ad378b9ec + + 'k8s.gcr.io/kube-controller-manager:v1.22.4': + sha1: 241924fa3dc4671fe6644402f7beb60028c02c71 + + 'k8s.gcr.io/kube-scheduler:v1.22.4': + sha1: 373e2939072b03cf5b1e115820b7fb6b749b0ebb + + 'k8s.gcr.io/kube-proxy:v1.22.4': + sha1: fecfb88509a430c29267a99b83f60f4a7c333583 + + 'k8s.gcr.io/coredns/coredns:v1.8.4': + sha1: 69c8e14ac3941fd5551ff22180be5f4ea2742d7f + + 'k8s.gcr.io/etcd:3.5.0-0': + sha1: 9d9ee2df54a201dcc9c7a10ea763b9a5dce875f1 + + 'k8s.gcr.io/pause:3.5': + sha1: bf3e3420df62f093f94c41d2b7a62b874dcbfc28 + + 'quay.io/coreos/flannel:v0.14.0-amd64': + sha1: cff47465996a51de4632b53abf1fca873f147027 + + 'quay.io/coreos/flannel:v0.14.0': + sha1: a487a36f7b31677e50e74b96b944f27fbce5ac13 + + 'calico/cni:v3.20.3': + sha1: 95e4cf79e92715b13e500a0efcfdb65590de1e04 + + 'calico/kube-controllers:v3.20.3': + sha1: 5769bae60830abcb3c5d97eb86b8f9938a587b2d + + 'calico/node:v3.20.3': + sha1: cc3c8727ad30b4850e8d0042681342a4f2351eff + + 'calico/pod2daemon-flexvol:v3.20.3': + sha1: 97c1b7ac90aa5a0f5c52e7f137549e598ff80f3e + + 'k8s.gcr.io/sig-storage/csi-attacher:v3.4.0': + sha1: f076bd75359c6449b965c48eb8bad96c6d40790d + + 'k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0': + sha1: 129eb73c8e118e5049fee3d273b2d477c547e080 + + 'k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0': + sha1: 2b45e5a3432cb89f3aec59584c1fa92c069e7a38 + + 'k8s.gcr.io/sig-storage/csi-resizer:v1.4.0': + sha1: ce5c57454254c195762c1f58e1d902d7e81ea669 + + 'k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1': + sha1: be1cf43617eea007629c0eb99149a99b6498f889 + + 'quay.io/ceph/ceph:v16.2.7': + sha1: fe9b7802c67e19111f83ffe4754ab62df66fd417 + allow_mismatch: true + + 'quay.io/cephcsi/cephcsi:v3.5.1': + sha1: 51dee9ea8ad76fb95ebd16f951e8ffaaaba95eb6 + + 'quay.io/csiaddons/k8s-sidecar:v0.2.1': + sha1: f0fd757436ac5075910c460c1991ff67c4774d09 + + 'quay.io/csiaddons/volumereplication-operator:v0.3.0': + sha1: d3cd17f14fcbf09fc6c8c2c5c0419f098f87a70f + + rook: + 'rook/ceph:v1.8.8': + sha1: f34039b17b18f5a855b096d48ff787b4013615e4 +""" + + +EXPECTED_VERBOSE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- kafka +- repository + +Features requested: +- filebeat +- firewall +- image-registry +- jmx-exporter +- kafka +- kafka-exporter +- node-exporter +- repository +- zookeeper +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_DASHBOARD_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- monitoring +- repository + +Features requested: +- filebeat +- firewall +- grafana +- image-registry +- node-exporter +- prometheus +- repository + +Dashboards to download: +- grafana_dashboard_10991 +- grafana_dashboard_11074 +- grafana_dashboard_179 +- grafana_dashboard_1860 +- grafana_dashboard_315 +- grafana_dashboard_405 +- grafana_dashboard_4279 +- grafana_dashboard_455 +- grafana_dashboard_6663 +- grafana_dashboard_7249 +- grafana_dashboard_7589 +- grafana_dashboard_789 +- grafana_dashboard_9628 +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_FILE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- kafka +- repository + +Features requested: +- filebeat +- firewall +- image-registry +- jmx-exporter +- kafka +- kafka-exporter +- node-exporter +- repository +- zookeeper + +Files to download: +- https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz +- https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz +- https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz +- https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz +- https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_IMAGE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- repository + +Features requested: +- filebeat +- firewall +- image-registry +- node-exporter +- repository + +Images to download: +- bitnami/pgpool:4.2.4 +- epiphanyplatform/keycloak:14.0.0 +- haproxy:2.2.2-alpine +- k8s.gcr.io/coredns/coredns:v1.8.0 +- k8s.gcr.io/etcd:3.4.13-0 +- k8s.gcr.io/kube-apiserver:v1.21.7 +- k8s.gcr.io/kube-apiserver:v1.22.4 +- k8s.gcr.io/kube-controller-manager:v1.21.7 +- k8s.gcr.io/kube-controller-manager:v1.22.4 +- k8s.gcr.io/kube-proxy:v1.21.7 +- k8s.gcr.io/kube-scheduler:v1.21.7 +- k8s.gcr.io/pause:3.4.1 +- kubernetesui/dashboard:v2.3.1 +- kubernetesui/metrics-scraper:v1.0.7 +- rabbitmq:3.8.9 +- registry:2.8.0 +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_IMAGE_NO_DOCUMENT_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- kafka +- rabbitmq +- repository + +Features requested: +- filebeat +- firewall +- image-registry +- jmx-exporter +- kafka +- kafka-exporter +- node-exporter +- rabbitmq +- repository +- zookeeper + +Images to download: +- registry:2.8.0 +-------------------------------------------------- +""" + + +EXPECTED_VERBOSE_K8S_AS_CLOUD_SERVICE_OUTPUT = """ +Manifest summary: +-------------------------------------------------- +Components requested: +- repository + +Features requested: +- filebeat +- firewall +- image-registry +- node-exporter +- repository + +Files to download: +- https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz +- https://charts.rook.io/release/rook-ceph-cluster-v1.8.8.tgz +- https://charts.rook.io/release/rook-ceph-v1.8.8.tgz +- https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz +-------------------------------------------------- +""" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py new file mode 100644 index 0000000000..04c0758aae --- /dev/null +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/data/manifest_reader.py @@ -0,0 +1,438 @@ +FEATURE_MAPPINGS = """ +--- +kind: configuration/feature-mappings +title: Feature mapping to roles +name: default +specification: + mappings: + kafka: + - zookeeper + - jmx-exporter + - kafka + - kafka-exporter + - node-exporter + - filebeat + - firewall + rabbitmq: + - rabbitmq + - node-exporter + - filebeat + - firewall + logging: + - logging + - opensearch-dashboards + - node-exporter + - filebeat + - firewall + load_balancer: + - haproxy + - node-exporter + - filebeat + - firewall + monitoring: + - prometheus + - grafana + - node-exporter + - filebeat + - firewall + postgresql: + - postgresql + - postgres-exporter + - node-exporter + - filebeat + - firewall + custom: + - repository + - image-registry + - kubernetes-master + - node-exporter + - filebeat + - rabbitmq + - postgresql + - prometheus + - grafana + - node-exporter + - logging + - firewall + - rook + single_machine: + - repository + - image-registry + - kubernetes-master + - helm + - applications + - rabbitmq + - postgresql + - firewall + - rook + kubernetes_master: + - kubernetes-master + - helm + - applications + - rook + - node-exporter + - filebeat + - firewall + kubernetes_node: + - kubernetes-node + - node-exporter + - filebeat + - firewall + opensearch: + - opensearch + - node-exporter + - filebeat + - firewall + repository: + - repository + - image-registry + - firewall + - filebeat + - node-exporter +version: 2.0.1dev +provider: azure +""" + + +INPUT_MANIFEST_FEATURE_MAPPINGS = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 2 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opensearch: + count: 0 +version: 2.0.1dev +{FEATURE_MAPPINGS} +""" + + +INPUT_MANIFEST_WITH_DASHBOARDS = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 1 + kafka: + count: 0 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opensearch: + count: 0 +version: 2.0.1dev +{FEATURE_MAPPINGS} +""" + + +INPUT_MANIFEST_WITH_IMAGES = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 0 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opensearch: + count: 0 +version: 2.0.1dev +{FEATURE_MAPPINGS} +--- +kind: configuration/image-registry +title: Epiphany image registry +name: default +specification: + description: Local registry with Docker images + registry_image: + name: registry:2.8.0 + file_name: registry-2.8.0.tar + images_to_load: + x86_64: + generic: + applications: + - name: epiphanyplatform/keycloak:14.0.0 + file_name: keycloak-14.0.0.tar + - name: rabbitmq:3.8.9 + file_name: rabbitmq-3.8.9.tar + - name: bitnami/pgpool:4.2.4 + file_name: pgpool-4.2.4.tar + kubernetes-master: + - name: kubernetesui/dashboard:v2.3.1 + file_name: dashboard-v2.3.1.tar + - name: kubernetesui/metrics-scraper:v1.0.7 + file_name: metrics-scraper-v1.0.7.tar + current: + haproxy: + - name: haproxy:2.2.2-alpine + file_name: haproxy-2.2.2-alpine.tar + kubernetes-master: + - name: k8s.gcr.io/kube-apiserver:v1.22.4 + file_name: kube-apiserver-v1.22.4.tar + - name: k8s.gcr.io/kube-controller-manager:v1.22.4 + file_name: kube-controller-manager-v1.22.4.tar + legacy: + kubernetes-master: + - name: k8s.gcr.io/kube-apiserver:v1.21.7 + file_name: kube-apiserver-v1.21.7.tar + - name: k8s.gcr.io/kube-controller-manager:v1.21.7 + file_name: kube-controller-manager-v1.21.7.tar + - name: k8s.gcr.io/kube-proxy:v1.21.7 + file_name: kube-proxy-v1.21.7.tar + - name: k8s.gcr.io/kube-scheduler:v1.21.7 + file_name: kube-scheduler-v1.21.7.tar + - name: k8s.gcr.io/coredns/coredns:v1.8.0 + file_name: coredns-v1.8.0.tar + - name: k8s.gcr.io/etcd:3.4.13-0 + file_name: etcd-3.4.13-0.tar + - name: k8s.gcr.io/pause:3.4.1 + file_name: pause-3.4.1.tar + aarch64: + generic: + applications: + - name: epiphanyplatform/keycloak:14.0.0 + file_name: keycloak-14.0.0.tar + - name: rabbitmq:3.8.9 + file_name: rabbitmq-3.8.9.tar + kubernetes-master: + - name: kubernetesui/dashboard:v2.3.1 + file_name: dashboard-v2.3.1.tar + - name: kubernetesui/metrics-scraper:v1.0.7 + file_name: metrics-scraper-v1.0.7.tar + current: + haproxy: + - name: haproxy:2.2.2-alpine + file_name: haproxy-2.2.2-alpine.tar + kubernetes-master: + - name: k8s.gcr.io/kube-apiserver:v1.22.4 + file_name: kube-apiserver-v1.22.4.tar + legacy: + kubernetes-master: + - name: k8s.gcr.io/kube-apiserver:v1.21.7 + file_name: kube-apiserver-v1.21.7.tar + - name: k8s.gcr.io/kube-scheduler:v1.21.7 + file_name: kube-scheduler-v1.21.7.tar + - name: k8s.gcr.io/coredns/coredns:v1.8.0 + file_name: coredns-v1.8.0.tar + - name: k8s.gcr.io/etcd:3.4.13-0 + file_name: etcd-3.4.13-0.tar + - name: k8s.gcr.io/pause:3.4.1 + file_name: pause-3.4.1.tar +version: 2.0.1dev +provider: any +""" + + +INPUT_MANIFEST_IMAGES_NO_DOCUMENT = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 2 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 1 + opensearch: + count: 0 +version: 2.0.1dev +{FEATURE_MAPPINGS} +""" + + +INPUT_MANIFEST_WITH_K8S_AS_CLOUD_SERVICE = f""" +kind: epiphany-cluster +title: Epiphany cluster Config +provider: any +name: default +specification: + name: new_cluster + admin_user: + name: operations + key_path: /shared/.ssh/epiphany-operations/id_rsa + cloud: + k8s_as_cloud_service: true + components: + repository: + count: 1 + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 0 + postgresql: + count: 0 + load_balancer: + count: 0 + rabbitmq: + count: 0 + opensearch: + count: 0 +version: 2.0.1dev +{FEATURE_MAPPINGS} +""" + + +EXPECTED_FEATURE_MAPPINGS = { + 'requested-components': ['kafka', 'repository'], + 'requested-features': ['filebeat', + 'firewall', + 'image-registry', + 'jmx-exporter', + 'kafka', + 'kafka-exporter', + 'node-exporter', + 'repository', + 'zookeeper'], + 'requested-images': [] +} + + +EXPECTED_FEATURE_MAPPINGS_WITH_DASHBOARDS = { + 'requested-components': ['monitoring', 'repository'], + 'requested-features': ['filebeat', + 'firewall', + 'grafana', + 'image-registry', + 'node-exporter', + 'prometheus', + 'repository'], + 'requested-images': [] +} + + +EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_X86_64 = { + 'requested-components': ['repository'], + 'requested-features': ['filebeat', + 'firewall', + 'image-registry', + 'node-exporter', + 'repository'], + 'requested-images': [ + 'bitnami/pgpool:4.2.4', + 'epiphanyplatform/keycloak:14.0.0', + 'haproxy:2.2.2-alpine', + 'k8s.gcr.io/coredns/coredns:v1.8.0', + 'k8s.gcr.io/etcd:3.4.13-0', + 'k8s.gcr.io/kube-apiserver:v1.21.7', + 'k8s.gcr.io/kube-apiserver:v1.22.4', + 'k8s.gcr.io/kube-controller-manager:v1.21.7', + 'k8s.gcr.io/kube-controller-manager:v1.22.4', + 'k8s.gcr.io/kube-proxy:v1.21.7', + 'k8s.gcr.io/kube-scheduler:v1.21.7', + 'k8s.gcr.io/pause:3.4.1', + 'kubernetesui/dashboard:v2.3.1', + 'kubernetesui/metrics-scraper:v1.0.7', + 'rabbitmq:3.8.9', + 'registry:2.8.0' + ] +} + + +EXPECTED_FEATURE_MAPPINGS_WITH_IMAGES_ARM64 = { + 'requested-components': ['repository'], + 'requested-features': ['filebeat', + 'firewall', + 'image-registry', + 'node-exporter', + 'repository'], + 'requested-images': [ + 'epiphanyplatform/keycloak:14.0.0', + 'haproxy:2.2.2-alpine', + 'k8s.gcr.io/coredns/coredns:v1.8.0', + 'k8s.gcr.io/etcd:3.4.13-0', + 'k8s.gcr.io/kube-apiserver:v1.21.7', + 'k8s.gcr.io/kube-apiserver:v1.22.4', + 'k8s.gcr.io/kube-scheduler:v1.21.7', + 'k8s.gcr.io/pause:3.4.1', + 'kubernetesui/dashboard:v2.3.1', + 'kubernetesui/metrics-scraper:v1.0.7', + 'rabbitmq:3.8.9', + 'registry:2.8.0' + ] +} diff --git a/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml b/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml index e9b26fd3ba..545613544e 100644 --- a/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml +++ b/ansible/playbooks/roles/repository/tasks/clean-up-epirepo.yml @@ -51,17 +51,45 @@ file: roles/image_registry/vars/main.yml name: image_registry_vars +- name: Define images to unpack + set_fact: + current_schema_images: "{{ image_registry_vars.specification.images_to_load[ansible_architecture].current }}" + generic_schema_images: "{{ image_registry_vars.specification.images_to_load[ansible_architecture].generic }}" + legacy_schema_images: "{{ image_registry_vars.specification.images_to_load[ansible_architecture].legacy }}" + +- name: Initialize image facts + set_fact: + current_images: [] + generic_images: [] + legacy_images: [] + +- name: Set list of current images to be loaded/pushed + set_fact: + current_images: "{{ current_schema_images | dict_to_list(only_values='True') | flatten }}" + +- name: Set list of generic images to be loaded/pushed + set_fact: + generic_images: "{{ generic_schema_images | dict_to_list(only_values='True') | flatten }}" + +- name: Set list of legacy images to be loaded/pushed + set_fact: + legacy_images: "{{ legacy_schema_images | dict_to_list(only_values='True') | flatten }}" + +- name: Merge current, legacy and generic images + set_fact: + all_images: >- + {{ current_images + generic_images + legacy_images }} + - name: Remove old images from epirepo file: state: absent path: "{{ _apache_epirepo_path }}/images/{{ item }}" vars: images_found: "{{ files_in_epirepo.results[1].files | map(attribute='path') | map('basename') }}" - images_to_load: "{{ image_registry_vars.specification.images_to_load[ansible_architecture] }}" - images_to_preserve: "{{ images_to_load | json_query('*[].file_name') + [ image_registry_vars.specification.registry_image.file_name ] }}" + images_to_preserve: "{{ all_images | json_query('[*].file_name') + [ image_registry_vars.specification.registry_image.file_name ] }}" # images to remove since they may have the same filename but different content (e.g. jboss/keycloak vs epiphanyplatform/keycloak), # to be optimized (checksums) - replaced_images: "{{ images_to_load | json_query('*[]') | selectattr('name', 'match', 'epiphanyplatform/') + replaced_images: "{{ all_images | json_query('[*]') | selectattr('name', 'match', 'epiphanyplatform/') | map(attribute='file_name') }}" images_to_remove: "{{ images_found | difference(images_to_preserve) + replaced_images }}" loop: "{{ images_to_remove }}" diff --git a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml index d6cb17c4f8..16b72da464 100644 --- a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml @@ -55,6 +55,12 @@ dest: "{{ download_requirements_dir }}/{{ item }}" loop: "{{ _files }}" + - name: Copy the manifest file + synchronize: + src: "{{ input_manifest_path }}" + dest: "{{ download_requirements_dir }}/manifest.yml" + when: not full_download and input_manifest_path + - name: Copy RedHat family specific download requirements file synchronize: src: "download-requirements/{{ _family_packages }}" diff --git a/ansible/playbooks/roles/repository/tasks/download-requirements.yml b/ansible/playbooks/roles/repository/tasks/download-requirements.yml index 534f23675f..32df762e37 100644 --- a/ansible/playbooks/roles/repository/tasks/download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/download-requirements.yml @@ -2,6 +2,25 @@ # download-requirements-done.flag file is used to avoid re-downloading requirements (to save time) # this is to be optimized in the future +- name: |- + Run download-requirements script, this can take a long time (optimized with manifest) + You can check progress on repository host with: journalctl -f -t download-requirements.py + shell: >- + set -o pipefail && + "{{ download_requirements_script }}" \ + /var/www/html/epirepo \ + "{{ download_requirements_os_name }}" \ + --manifest "{{ download_requirements_manifest }}" \ + --no-logfile \ + --repos-backup-file /var/tmp/enabled-system-repos.tar \ + --verbose |& + tee >(systemd-cat --identifier=download-requirements.py) + args: + executable: /bin/bash + when: + - not full_download + - input_manifest_path + - name: |- Run download-requirements script, this can take a long time You can check progress on repository host with: journalctl -f -t download-requirements.py @@ -10,11 +29,13 @@ "{{ download_requirements_script }}" \ /var/www/html/epirepo \ "{{ download_requirements_os_name }}" \ + --no-logfile \ --repos-backup-file /var/tmp/enabled-system-repos.tar \ - --no-logfile |& + --verbose |& tee >(systemd-cat --identifier=download-requirements.py) args: executable: /bin/bash + when: full_download or not input_manifest_path - name: Create flag file to not re-download requirements next time file: diff --git a/ansible/playbooks/roles/repository/tasks/setup.yml b/ansible/playbooks/roles/repository/tasks/setup.yml index 545a8ee568..d4dd2203d9 100644 --- a/ansible/playbooks/roles/repository/tasks/setup.yml +++ b/ansible/playbooks/roles/repository/tasks/setup.yml @@ -123,4 +123,6 @@ - name: Include Helm repository creation include_tasks: "create-helm-repo.yml" - when: inventory_hostname == target_repository_hostnames[0] + when: + - inventory_hostname == target_repository_hostnames[0] + - (groups['kubernetes_master'] is defined and inventory_hostname in groups['kubernetes_master']) diff --git a/cli/epicli.py b/cli/epicli.py index 81b1265110..d16ca94ed2 100644 --- a/cli/epicli.py +++ b/cli/epicli.py @@ -213,6 +213,8 @@ def apply_parser(subparsers): help='Number of pings after which Ansible will fail.') optional.add_argument('--ansible-forks', dest='ansible_forks', type=int, required=False, action='store', default=10, help='Sets the number of forks in ansible.cfg.') + optional.add_argument('--full-download', dest='full_download', required=False, action='store_true', default=False, + help='When used epicli will download all the available requirements for each feature supported.') sub_parser._action_groups.append(optional) def run_apply(args): @@ -306,6 +308,8 @@ def splitarg(arg): help='Number of pings after which Ansible will fail.') optional.add_argument('--ansible-forks', dest='ansible_forks', type=int, required=False, action='store', default=10, help='Sets the number of forks in ansible.cfg.') + optional.add_argument('--full-download', dest='full_download', required=False, action='store_true', default=False, + help='When used epicli will download all the available requirements for each feature supported.') sub_parser._action_groups.append(optional) def run_upgrade(args): diff --git a/cli/src/Config.py b/cli/src/Config.py index 0f3857c113..3fa1a3ed4d 100644 --- a/cli/src/Config.py +++ b/cli/src/Config.py @@ -1,5 +1,6 @@ import os from os.path import expanduser +from pathlib import Path from typing import Dict, List @@ -32,6 +33,8 @@ def __init__(self): self._log_count = 10 self._log_type = 'plain' + self._full_download: bool = False + self._input_manifest_path: Path = None self._validate_certs = True self._debug = 0 self._auto_approve = False @@ -40,6 +43,22 @@ def __init__(self): self._upgrade_components = [] self._vault_password_location = os.path.join(expanduser("~"), '.epicli/vault.cfg') + @property + def full_download(self) -> bool: + return self._full_download + + @full_download.setter + def full_download(self, full_download: bool): + self._full_download = full_download + + @property + def input_manifest_path(self) -> Path: + return self._input_manifest_path + + @input_manifest_path.setter + def input_manifest_path(self, input_manifest_path: Path): + self._input_manifest_path = input_manifest_path + @property def docker_cli(self): return self._docker_cli diff --git a/cli/src/ansible/AnsibleInventoryCreator.py b/cli/src/ansible/AnsibleInventoryCreator.py index d964e28d01..07c580f05a 100644 --- a/cli/src/ansible/AnsibleInventoryCreator.py +++ b/cli/src/ansible/AnsibleInventoryCreator.py @@ -43,12 +43,12 @@ def get_inventory(self): return self.group_duplicated(inventory) def get_roles_for_feature(self, component_key): - features_map = select_single(self.config_docs, lambda x: x.kind == 'configuration/feature-mapping') - return features_map.specification.roles_mapping[component_key] + features_map = select_single(self.config_docs, lambda x: x.kind == 'configuration/feature-mappings') + return features_map.specification.mappings[component_key] def get_available_roles(self): - features_map = select_single(self.config_docs, lambda x: x.kind == 'configuration/feature-mapping') - return features_map.specification.available_roles + features = select_single(self.config_docs, lambda x: x.kind == 'configuration/features') + return features.specification.features def get_enabled_roles(self): roles = self.get_available_roles() diff --git a/cli/src/ansible/AnsibleVarsGenerator.py b/cli/src/ansible/AnsibleVarsGenerator.py index 63f7d55db3..502baa5627 100644 --- a/cli/src/ansible/AnsibleVarsGenerator.py +++ b/cli/src/ansible/AnsibleVarsGenerator.py @@ -141,15 +141,19 @@ def write_role_manifest_vars(self, ansible_dir, role, kind): self.write_role_vars(ansible_dir, role, document, vars_file_name='manifest.yml') def populate_group_vars(self, ansible_dir): + input_manifest_path: str = str(Config().input_manifest_path.absolute()) if Config().input_manifest_path else '' + main_vars = ObjDict() main_vars['admin_user'] = self.cluster_model.specification.admin_user - main_vars['validate_certs'] = Config().validate_certs - main_vars['offline_requirements'] = Config().offline_requirements - main_vars['wait_for_pods'] = Config().wait_for_pods + main_vars['epiphany_version'] = VERSION + main_vars['input_manifest_path'] = input_manifest_path main_vars['is_upgrade_run'] = self.is_upgrade_run + main_vars['offline_requirements'] = Config().offline_requirements main_vars['roles_with_generated_vars'] = sorted(self.roles_with_generated_vars) main_vars['upgrade_components'] = Config().upgrade_components - main_vars['epiphany_version'] = VERSION + main_vars['validate_certs'] = Config().validate_certs + main_vars['wait_for_pods'] = Config().wait_for_pods + main_vars['full_download'] = Config().full_download # Consider to move this to the provider level. if self.cluster_model.provider != 'any': diff --git a/cli/src/commands/Apply.py b/cli/src/commands/Apply.py index 49b5466ee4..b29d46867f 100644 --- a/cli/src/commands/Apply.py +++ b/cli/src/commands/Apply.py @@ -1,6 +1,9 @@ import os import sys +from pathlib import Path +from typing import Dict +from cli.src.Config import Config from cli.src.ansible.AnsibleRunner import AnsibleRunner from cli.src.helpers.build_io import (get_build_path, get_inventory_path, get_manifest_path, load_inventory, @@ -42,14 +45,8 @@ def __init__(self, input_data): self.infrastructure_docs = [] self.all_docs = [] - - def __enter__(self): - return self - - - def __exit__(self, exc_type, exc_value, traceback): - pass - + Config().full_download = input_data.full_download + Config().input_manifest_path = Path(self.file) def load_documents(self): # Load the input docs from the input @@ -213,23 +210,26 @@ def assert_no_master_downscale(self): raise Exception("ControlPlane downscale is not supported yet. Please revert your 'kubernetes_master' count to previous value or increase it to scale up Kubernetes.") + def __load_configuration_doc(self, kind: str) -> Dict: + doc = select_first(self.input_docs, lambda x: x.kind == kind) + if not doc: + return load_schema_obj(schema_types.DEFAULT, 'common', kind) + + with DefaultMerger([doc]) as doc_merger: + return doc_merger.run()[0] + def assert_no_postgres_nodes_number_change(self): - feature_mapping = select_first(self.input_docs, lambda x: x.kind == 'configuration/feature-mapping') - if feature_mapping: - with DefaultMerger([feature_mapping]) as doc_merger: - feature_mapping = doc_merger.run() - feature_mapping = feature_mapping[0] - else: - feature_mapping = load_schema_obj(schema_types.DEFAULT, 'common', 'configuration/feature-mapping') + feature_mappings = self.__load_configuration_doc('configuration/feature-mappings') + features = self.__load_configuration_doc('configuration/features') components = self.cluster_model.specification.components if self.inventory: next_postgres_node_count = 0 prev_postgres_node_count = len(self.inventory.list_hosts(pattern='postgresql')) - postgres_available = [x for x in feature_mapping.specification.available_roles if x.name == 'postgresql'] + postgres_available = [x for x in features.specification.features if x.name == 'postgresql'] if postgres_available[0].enabled: - for key, roles in feature_mapping.specification.roles_mapping.items(): - if ('postgresql') in roles and key in components: + for key, features in feature_mappings.specification.mappings.items(): + if ('postgresql') in features and key in components: next_postgres_node_count = next_postgres_node_count + components[key].count if prev_postgres_node_count > 0 and prev_postgres_node_count != next_postgres_node_count: diff --git a/cli/src/commands/Init.py b/cli/src/commands/Init.py index a69cefe5a5..f11db038aa 100644 --- a/cli/src/commands/Init.py +++ b/cli/src/commands/Init.py @@ -49,6 +49,9 @@ def init(self): config_docs = config_appender.run() docs = [*config_docs, *infra_docs] + else: + with ConfigurationAppender(docs) as config_appender: + config_appender.add_feature_mappings() # set the provider and version for all docs for doc in docs: diff --git a/cli/src/commands/Upgrade.py b/cli/src/commands/Upgrade.py index b8a685493c..585f1a1d02 100644 --- a/cli/src/commands/Upgrade.py +++ b/cli/src/commands/Upgrade.py @@ -1,7 +1,9 @@ import os import re import time +from pathlib import Path +from cli.src.Config import Config from cli.src.ansible.AnsibleCommand import AnsibleCommand from cli.src.ansible.AnsibleRunner import AnsibleRunner from cli.src.helpers.build_io import copy_files_recursively @@ -23,6 +25,9 @@ def __init__(self, input_data): self.input_docs = [] self.ping_retries: int = input_data.ping_retries + Config().full_download = input_data.full_download + Config().input_manifest_path = Path(self.file) if self.file else None + def __enter__(self): super().__enter__() return self diff --git a/cli/src/schema/ConfigurationAppender.py b/cli/src/schema/ConfigurationAppender.py index 40f00f76c2..d0d1de0bc6 100644 --- a/cli/src/schema/ConfigurationAppender.py +++ b/cli/src/schema/ConfigurationAppender.py @@ -1,3 +1,5 @@ +from typing import Callable, Dict, List + from cli.src.helpers.config_merger import merge_with_defaults from cli.src.helpers.data_loader import load_schema_obj, schema_types from cli.src.helpers.doc_list_helpers import select_first, select_single @@ -6,45 +8,60 @@ class ConfigurationAppender(Step): - REQUIRED_DOCS = ['epiphany-cluster', 'configuration/feature-mapping', 'configuration/shared-config'] + REQUIRED_DOCS = ['epiphany-cluster', + 'configuration/features', + 'configuration/feature-mappings', + 'configuration/shared-config'] def __init__(self, input_docs): super().__init__(__name__) - self.cluster_model = select_single(input_docs, lambda x: x.kind == 'epiphany-cluster') - self.input_docs = input_docs + self.__cluster_model: Dict = select_single(input_docs, lambda x: x.kind == 'epiphany-cluster') + self.__input_docs: List[Dict] = input_docs - def run(self): - configuration_docs = [] + def __append_config(self, config_docs: List[Dict], document: Dict): + document['version'] = VERSION + config_docs.append(document) + + def __add_doc(self, config_docs: List[Dict], document_kind: str): + doc = select_first(self.__input_docs, lambda x, kind=document_kind: x.kind == kind) + if doc is None: + doc = load_schema_obj(schema_types.DEFAULT, 'common', document_kind) + self.logger.info(f'Adding: {doc.kind}') + + self.__append_config(config_docs, doc) + + def __feature_selector(self, feature_key: str, config_selector: str) -> Callable: + return lambda x, key=feature_key, selector=config_selector: x.kind == f'configuration/{key}' and x.name == selector - def append_config(doc): - doc['version'] = VERSION - configuration_docs.append(doc) + def add_feature_mappings(self): + feature_mappings: List[Dict] = [] + self.__add_doc(feature_mappings, 'configuration/feature-mappings') + + if feature_mappings is not None: + self.__input_docs.append(feature_mappings[0]) + + def run(self): + configuration_docs: List[Dict] = [] for document_kind in ConfigurationAppender.REQUIRED_DOCS: - doc = select_first(self.input_docs, lambda x: x.kind == document_kind) - if doc is None: - doc = load_schema_obj(schema_types.DEFAULT, 'common', document_kind) - self.logger.info("Adding: " + doc.kind) - append_config(doc) - else: - append_config(doc) - - for component_key, component_value in self.cluster_model.specification.components.items(): + self.__add_doc(configuration_docs, document_kind) + + for component_key, component_value in self.__cluster_model.specification.components.items(): if component_value.count < 1: continue - features_map = select_first(configuration_docs, lambda x: x.kind == 'configuration/feature-mapping') + feature_mappings = select_first(configuration_docs, lambda x: x.kind == 'configuration/feature-mappings') config_selector = component_value.configuration - for feature_key in features_map.specification.roles_mapping[component_key]: - config = select_first(self.input_docs, lambda x: x.kind == 'configuration/' + feature_key and x.name == config_selector) - if config is not None: - append_config(config) - if config is None: - config = select_first(configuration_docs, lambda - x: x.kind == 'configuration/' + feature_key and x.name == config_selector) - if config is None: - config = merge_with_defaults('common', 'configuration/' + feature_key, config_selector, self.input_docs) - self.logger.info("Adding: " + config.kind) - append_config(config) + for feature_key in feature_mappings.specification.mappings[component_key]: + first_input_docs_config = select_first(self.__input_docs, self.__feature_selector(feature_key, config_selector)) + if first_input_docs_config is not None: + self.__append_config(configuration_docs, first_input_docs_config) + else: + first_config = select_first(configuration_docs, self.__feature_selector(feature_key, config_selector)) + + if first_config is None: + merged_config = merge_with_defaults('common', f'configuration/{feature_key}', config_selector, self.__input_docs) + self.logger.info(f'Adding: {merged_config.kind}') + self.__append_config(configuration_docs, merged_config) return configuration_docs diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 09bf8c05ab..86b6dbfb68 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -7,14 +7,15 @@ - [#2932](https://github.com/epiphany-platform/epiphany/issues/2932) - Support `epicli upgrade` for RHEL/AlmaLinux 8 - [#3057](https://github.com/epiphany-platform/epiphany/issues/3057) - Additional AWS authentication option - [#3101](https://github.com/epiphany-platform/epiphany/issues/3101) - Add support for ARM architecture for AlmaLinux 8 +- [#3105](https://github.com/epiphany-platform/epiphany/issues/3105) - Add manifest file parsing +- [#3131](https://github.com/epiphany-platform/epiphany/issues/3131) - Optimize Grafana dashboards downloading +- [#3116](https://github.com/epiphany-platform/epiphany/issues/3116) - Optimize files downloading +- [#3106](https://github.com/epiphany-platform/epiphany/issues/3106) - Add image-registry configuration reading ### Fixed - [#3164](https://github.com/epiphany-platform/epiphany/issues/3164) - Specify version and allow containerd.io package downgrade in haproxy_runc role - [#3179](https://github.com/epiphany-platform/epiphany/issues/3179) - terraform fails when `use_network_security_groups` is set to `false` - -### Fixed - - [#3165](https://github.com/epiphany-platform/epiphany/issues/3165) - download-requirements.py may fail due to expired certificate ### Updated @@ -23,6 +24,7 @@ - [#2982](https://github.com/epiphany-platform/epiphany/issues/2982) - Using AKS and EKS Terraform configuration directly with Epiphany. - [#2870](https://github.com/epiphany-platform/epiphany/issues/2870) - OpenDistro for ElasticSearch replaced by OpenSearch - [#3163](https://github.com/epiphany-platform/epiphany/issues/3163) - Upgrade Python dependencies +- [#3097](https://github.com/epiphany-platform/epiphany/issues/3097) - Split available_roles and roles_mapping into separate yaml documents ### Deprecated @@ -34,6 +36,8 @@ ### Breaking changes +- Schema `configuration/feature-mapping` changed. The document was splitted into two separate docs `configuration/features` and `configuration/feature-mappings`. + - AWS credentials configuration parameters are renamed from `specification.cloud.credentials.key` and `specification.cloud.credentials.secret` to `specification.cloud.credentials.access_key_id` and `specification.cloud.credentials.secret_access_key`. ## [2.0.0] 2022-05-09 @@ -90,7 +94,7 @@ - [#2803](https://github.com/epiphany-platform/epiphany/issues/2803) - Refactor: rename 'kafka_var' setting - [#2995](https://github.com/epiphany-platform/epiphany/issues/2995) - Update expired RHUI client certificate before installing any RHEL packages - [#3049](https://github.com/epiphany-platform/epiphany/issues/3049) - HAProxy upgrade fails on re-run trying to remove haproxy_exporter -- [#3006](https://github.com/epiphany-platform/epiphany/issues/3006) - install 'containerd.io=1.4.12-*' failed, when upgrade from v1.3.0 to 2.0.0dev +- [#3006](https://github.com/epiphany-platform/epiphany/issues/3006) - install `containerd.io=1.4.12-*` failed, when upgrade from v1.3.0 to 2.0.0dev - [#3065](https://github.com/epiphany-platform/epiphany/issues/3065) - Flag `delete_os_disk_on_termination` has no effect when removing cluster - [#3153](https://github.com/epiphany-platform/epiphany/issues/3153) - AlmaLinux 8.5 installation fails resolving dependencies diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index fe5bfca9de..aff0aa2160 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -57,10 +57,9 @@ Disable: 2. Prepend `kubernetes_master` mapping (or any other mapping if you don't deploy Kubernetes) with: ```yaml - kind: configuration/feature-mapping + kind: configuration/feature-mappings specification: - ... - roles_mapping: + mappings: ... kubernetes_master: - repository @@ -290,12 +289,12 @@ specification: kubernetes_node: count: 2 --- -kind: configuration/feature-mapping -title: "Feature mapping to roles" +kind: configuration/feature-mappings +title: "Feature mapping to components" provider: name: default specification: - roles_mapping: + mappings: kubernetes_master: - repository - image-registry @@ -671,40 +670,44 @@ specification: Epiphany gives you the ability to define custom components. This allows you to define a custom set of roles for a component you want to use in your cluster. It can be useful when you for example want to maximize usage of the available machines you have at your disposal. -The first thing you will need to do is define it in the `configuration/feature-mapping` configuration. To get this configuration you can run `epicli init ... --full` command. In the `available_roles` roles section you can see all the available roles that Epiphany provides. The `roles_mapping` is where all the Epiphany components are defined and were you need to add your custom components. +The first thing you will need to do is define it in the `configuration/features` and the `configuration/feature-mappings` configurations. To get these configurations you can run `epicli init ... --full` command. In the `configuration/features` doc you can see all the available features that Epiphany provides. The `configuration/feature-mappings` doc is where all the Epiphany components are defined and where you can add your custom components. -Below are parts of an example `configuration/feature-mapping` were we define a new `single_machine_new` component. We want to use Kafka instead of RabbitMQ and don`t need applications and postgres since we don't want a Keycloak deployment: +Below are parts of an example `configuration/features` and `configuration/feature-mappings` docs where we define a new `single_machine_new` component. We want to use Kafka instead of RabbitMQ and don't need applications and postgres since we don't want a Keycloak deployment: ```yaml -kind: configuration/feature-mapping -title: Feature mapping to roles +kind: configuration/features +title: "Features to be enabled/disabled" name: default specification: - available_roles: # All entries here represent the available roles within Epiphany - - name: repository - enabled: yes - - name: firewall - enabled: yes - - name: image-registry - ... - roles_mapping: # All entries here represent the default components provided with Epiphany - ... + features: # All entries here represent the available features within Epiphany + - name: repository + enabled: yes + - name: firewall + enabled: yes + - name: image-registry + ... +--- +kind: configuration/feature-mappings +title: "Feature mapping to components" +name: default +specification: + mappings: # All entries here represent the default components provided with Epiphany single_machine: - - repository - - image-registry - - kubernetes-master - - applications - - rabbitmq - - postgresql - - firewall + - repository + - image-registry + - kubernetes-master + - applications + - rabbitmq + - postgresql + - firewall # Below is the new single_machine_new definition single_machine_new: - - repository - - image-registry - - kubernetes-master - - kafka - - firewall - ... + - repository + - image-registry + - kubernetes-master + - kafka + - firewall + ... ``` Once defined the new `single_machine_new` can be used inside the `epiphany-cluster` configuration: diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 3b84bfb861..c0df337025 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -490,11 +490,11 @@ By default, OpenSearch Dashboards ( previously Kibana component ) is deployed o for `opensearch` component you have to modify feature mapping. Use below configuration in your manifest: ```yaml -kind: configuration/feature-mapping -title: "Feature mapping to roles" +kind: configuration/feature-mappings +title: "Feature mapping to components" name: default specification: - roles_mapping: + mappings: opensearch: - node-exporter - filebeat diff --git a/docs/home/howto/KUBERNETES.md b/docs/home/howto/KUBERNETES.md index c3d3d452b1..cc4ea04daf 100644 --- a/docs/home/howto/KUBERNETES.md +++ b/docs/home/howto/KUBERNETES.md @@ -143,15 +143,15 @@ specification: count: 2 ``` -2. Enable `applications` in feature-mapping in initial configuration manifest. +2. Enable `applications` in `configuration/features` in initial configuration manifest. ```yaml --- -kind: configuration/feature-mapping -title: Feature mapping to roles +kind: configuration/features +title: "Features to be enabled/disabled" name: default specification: - available_roles: + features: - name: applications enabled: true ``` diff --git a/docs/home/howto/MODULES.md b/docs/home/howto/MODULES.md new file mode 100644 index 0000000000..bf5adfbc80 --- /dev/null +++ b/docs/home/howto/MODULES.md @@ -0,0 +1,403 @@ +# Modules + +## Introduction + +In version 0.8 of Epiphany we introduced modules. Modularization of Epiphany environment will result in: + +* smaller code bases for separate areas, +* simpler and faster test process, +* interchangeability of elements providing similar functionality (eg.: different Kubernetes providers), +* faster and more focused release cycle. + +Those and multiple other factors (eg.: readability, reliability) influence this direction of changes. + +## User point of view + +From a user point of view, there will be no significant changes in the nearest future as it will be still possible to install Epiphany "classic way" so with a single `epicli` configuration using a whole codebase as a monolith. + +For those who want to play with new features, or will need newly introduced possibilities, there will be a short transition period which we consider as a kind of "preview stage". In this period there will be a need to run each module separately by hand in the following order: + +* moduleA init +* moduleA plan +* moduleA apply +* moduleB init +* moduleB plan +* moduleB apply +* ... + +Init, plan and apply phases explanation you'll find in next sections of this document. Main point is that dependent modules have to be executed one after another during this what we called "preview stage". Later, with next releases there will be separate mechanism introduced to orchestrate modules dependencies and their consecutive execution. + +## New scenarios + +In 0.8 we offer the possibility to use AKS or EKS as Kubernetes providers. That is introduced with modules mechanism, so we launched the first four modules: + +* [Azure Basic Infrastructure](https://github.com/epiphany-platform/m-azure-basic-infrastructure) (AzBI) module +* [Azure AKS](https://github.com/epiphany-platform/m-azure-kubernetes-service) (AzKS) module +* [AWS Basic Infrastructure](https://github.com/epiphany-platform/m-aws-basic-infrastructure) (AwsBI) module +* [AWS EKS](https://github.com/epiphany-platform/m-aws-kubernetes-service) (AwsKS) module + +Those 4 modules together with the classic Epiphany used with `any` provider allow replacing of on-prem Kubernetes cluster with managed Kubernetes services. + +As it might be already visible there are 2 paths provided: + +* Azure related, using AzBI and AzKS modules, +* AWS related, using AwsBI and AwsKS modules. + +Those "... Basic Infrastructure" modules are responsible to provide basic cloud resources (eg.: resource groups, virtual networks, subnets, virtual machines, network security rules, routing, ect.) which will be used by next modules. So in this case, those are "... KS modules" meant to provide managed Kubernetes services. They use resources provided by basic infrastructure modules (eg.: subnets or resource groups) and instantiate managed Kubernetes services provided by cloud providers. The last element in both those cloud provider related paths is classic Epiphany installed on top of resources provided by those modules using `any` provider. + +## Hands-on + +In each module, we provided a guide on how to use the module. Please refer: + +* [Azure Basic Infrastructure](https://github.com/epiphany-platform/m-azure-basic-infrastructure/blob/develop/README.md) (AzBI) module +* [Azure AKS](https://github.com/epiphany-platform/m-azure-kubernetes-service/blob/develop/README.md) (AzKS) module +* [AWS Basic Infrastructure](https://github.com/epiphany-platform/m-aws-basic-infrastructure/blob/develop/README.md) (AwsBI) module +* [AWS EKS](https://github.com/epiphany-platform/m-aws-kubernetes-service/blob/develop/README.md) (AwsKS) module + +After deployment of EKS or AKS, you can perform Epiphany installation on top of it. + +### Install Epiphany on top of AzKS or AwsKS + +NOTE - Default OS users: + +```yaml +Azure: + redhat: ec2-user + ubuntu: operations +AWS: + redhat: ec2-user + ubuntu: ubuntu +``` + +* Create Epiphany cluster config file in `/tmp/shared/epi.yml` + Example: + + ```yaml + kind: epiphany-cluster + title: Epiphany cluster Config + name: your-cluster-name # <----- make unified with other places and build directory name + provider: any # <----- use "any" provider + specification: + name: your-cluster-name # <----- make unified with other places and build directory name + admin_user: + name: operations # <----- make sure os-user is correct + key_path: /tmp/shared/vms_rsa # <----- use generated key file + cloud: + k8s_as_cloud_service: true # <----- make sure that flag is set, as it indicates usage of a managed Kubernetes service + components: + repository: + count: 1 + machines: + - default-epiphany-modules-test-all-0 # <----- make sure that it is correct VM name + kubernetes_master: + count: 0 + kubernetes_node: + count: 0 + logging: + count: 0 + monitoring: + count: 0 + kafka: + count: 0 + postgresql: + count: 1 + machines: + - default-epiphany-modules-test-all-1 # <----- make sure that it is correct VM name + load_balancer: + count: 0 + rabbitmq: + count: 0 + --- + kind: configuration/feature-mappings + title: "Feature mapping to components" + name: your-cluster-name # <----- make unified with other places and build directory name + provider: any + specification: + mappings: + repository: + - repository + - image-registry + - firewall + - filebeat + - node-exporter + - applications + --- + kind: infrastructure/machine + name: default-epiphany-modules-test-all-0 + provider: any + specification: + hostname: epiphany-modules-test-all-0 + ip: 12.34.56.78 # <----- put here public IP attached to machine + --- + kind: infrastructure/machine + name: default-epiphany-modules-test-all-1 + provider: any + specification: + hostname: epiphany-modules-test-all-1 + ip: 12.34.56.78 # <----- put here public IP attached to machine + --- + kind: configuration/repository + title: "Epiphany requirements repository" + name: default + specification: + description: "Local repository of binaries required to install Epiphany" + download_done_flag_expire_minutes: 120 + apache_epirepo_path: "/var/www/html/epirepo" + teardown: + disable_http_server: true + remove: + files: false + helm_charts: false + images: false + packages: false + provider: any + --- + kind: configuration/postgresql + title: PostgreSQL + name: default + specification: + config_file: + parameter_groups: + - name: CONNECTIONS AND AUTHENTICATION + subgroups: + - name: Connection Settings + parameters: + - name: listen_addresses + value: "'*'" + comment: listen on all addresses + - name: Security and Authentication + parameters: + - name: ssl + value: 'off' + comment: to have the default value also on Ubuntu + - name: RESOURCE USAGE (except WAL) + subgroups: + - name: Kernel Resource Usage + parameters: + - name: shared_preload_libraries + value: AUTOCONFIGURED + comment: set by automation + - name: ERROR REPORTING AND LOGGING + subgroups: + - name: Where to Log + parameters: + - name: log_directory + value: "'/var/log/postgresql'" + comment: to have standard location for Filebeat and logrotate + - name: log_filename + value: "'postgresql.log'" + comment: to use logrotate with common configuration + - name: WRITE AHEAD LOG + subgroups: + - name: Settings + parameters: + - name: wal_level + value: replica + when: replication + - name: Archiving + parameters: + - name: archive_mode + value: 'on' + when: replication + - name: archive_command + value: "'test ! -f /dbbackup/{{ inventory_hostname }}/backup/%f &&\ + \ gzip -c < %p > /dbbackup/{{ inventory_hostname }}/backup/%f'" + when: replication + - name: REPLICATION + subgroups: + - name: Sending Server(s) + parameters: + - name: max_wal_senders + value: 10 + comment: maximum number of simultaneously running WAL sender processes + when: replication + - name: wal_keep_segments + value: 34 + comment: number of WAL files held for standby servers + when: replication + extensions: + pgaudit: + enabled: false + shared_preload_libraries: + - pgaudit + config_file_parameters: + log_connections: 'off' + log_disconnections: 'off' + log_statement: 'none' + log_line_prefix: "'%m [%p] %q%u@%d,host=%h '" + pgaudit.log: "'write, function, role, ddl' # 'misc_set' is not supported for\ + \ PG 10" + pgaudit.log_catalog: 'off # to reduce overhead of logging' + pgaudit.log_relation: 'on # separate log entry for each relation' + pgaudit.log_statement_once: 'off' + pgaudit.log_parameter: 'on' + replication: + replication_user_name: epi_repmgr + replication_user_password: PASSWORD_TO_CHANGE + privileged_user_name: epi_repmgr_admin + privileged_user_password: PASSWORD_TO_CHANGE + repmgr_database: epi_repmgr + shared_preload_libraries: + - repmgr + logrotate: + config: |- + /var/log/postgresql/postgresql*.log { + maxsize 10M + daily + rotate 6 + copytruncate + # delaycompress is for Filebeat + delaycompress + compress + notifempty + missingok + su root root + nomail + # to have multiple unique filenames per day when dateext option is set + dateformat -%Y%m%dH%H + } + provider: any + --- + kind: configuration/applications + title: "Kubernetes Applications Config" + name: default + specification: + applications: + - name: rabbitmq + enabled: false + image_path: rabbitmq:3.8.3 + use_local_image_registry: false + service: + name: rabbitmq-cluster + port: 30672 + management_port: 31672 + replicas: 2 + namespace: queue + rabbitmq: + plugins: + - rabbitmq_management + - rabbitmq_management_agent + policies: + - name: ha-policy2 + pattern: ".*" + definitions: + ha-mode: all + custom_configurations: + - name: vm_memory_high_watermark.relative + value: 0.5 + cluster: + - name: auth-service + enabled: false + image_path: jboss/keycloak:9.0.0 + use_local_image_registry: false + service: + name: as-testauthdb + port: 30104 + replicas: 2 + namespace: namespace-for-auth + admin_user: auth-service-username + admin_password: PASSWORD_TO_CHANGE + database: + name: auth-database-name + user: auth-db-user + password: PASSWORD_TO_CHANGE + - name: pgpool + enabled: true + image: + path: bitnami/pgpool:4.1.1-debian-10-r29 + debug: false + use_local_image_registry: false + namespace: postgres-pool + service: + name: pgpool + port: 5432 + replicas: 3 + pod_spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - pgpool + topologyKey: kubernetes.io/hostname + nodeSelector: {} + tolerations: {} + resources: + limits: + memory: 176Mi + requests: + cpu: 250m + memory: 176Mi + pgpool: + env: + PGPOOL_BACKEND_NODES: autoconfigured + PGPOOL_POSTGRES_USERNAME: epi_pgpool_postgres_admin + PGPOOL_SR_CHECK_USER: epi_pgpool_sr_check + PGPOOL_ADMIN_USERNAME: epi_pgpool_admin + PGPOOL_ENABLE_LOAD_BALANCING: true + PGPOOL_MAX_POOL: 4 + PGPOOL_POSTGRES_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_postgres_password + PGPOOL_SR_CHECK_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_sr_check_password + PGPOOL_ADMIN_PASSWORD_FILE: /opt/bitnami/pgpool/secrets/pgpool_admin_password + secrets: + pgpool_postgres_password: PASSWORD_TO_CHANGE + pgpool_sr_check_password: PASSWORD_TO_CHANGE + pgpool_admin_password: PASSWORD_TO_CHANGE + pgpool_conf_content_to_append: | + #------------------------------------------------------------------------------ + # CUSTOM SETTINGS (appended by Epiphany to override defaults) + #------------------------------------------------------------------------------ + # num_init_children = 32 + connection_life_time = 900 + reserved_connections = 1 + pool_hba_conf: autoconfigured + - name: pgbouncer + enabled: true + image_path: brainsam/pgbouncer:1.12 + init_image_path: bitnami/pgpool:4.1.1-debian-10-r29 + use_local_image_registry: false + namespace: postgres-pool + service: + name: pgbouncer + port: 5432 + replicas: 2 + resources: + requests: + cpu: 250m + memory: 128Mi + limits: + cpu: 500m + memory: 128Mi + pgbouncer: + env: + DB_HOST: pgpool.postgres-pool.svc.cluster.local + DB_LISTEN_PORT: 5432 + LISTEN_ADDR: "*" + LISTEN_PORT: 5432 + AUTH_FILE: "/etc/pgbouncer/auth/users.txt" + AUTH_TYPE: md5 + MAX_CLIENT_CONN: 150 + DEFAULT_POOL_SIZE: 25 + RESERVE_POOL_SIZE: 25 + POOL_MODE: transaction + provider: any + ``` + +* Run `epicli` tool to install Epiphany: + + ```shell + epicli --auto-approve apply --file='/tmp/shared/epi.yml' --vault-password='secret' + ``` + + This will install PostgreSQL on one of the machines and configure PgBouncer, Pgpool and additional services to manage database connections. + + Please make sure you disable applications that you don't need. Also, you can enable standard Epiphany services like Kafka or RabbitMQ, by increasing the number of virtual machines in the basic infrastructure config and assigning them to Epiphany components you want to use. + + If you would like to deploy custom resources into managed Kubernetes, then the standard kubeconfig yaml document can be found inside the shared state file (you should be able to use vendor tools as well to get it). + + We highly recommend using the `Ingress` resource in Kubernetes to allow access to web applications inside the cluster. Since it's managed Kubernetes and fully supported by the cloud platform, the classic HAProxy load-balancer solution seems to be deprecated here. diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 49db4473f5..ca3d700944 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -246,12 +246,12 @@ specification: prefix: 'prefix' title: Epiphany cluster Config --- -kind: configuration/feature-mapping -title: Feature mapping to roles +kind: configuration/feature-mappings +title: "Feature mapping to components" provider: azure name: default specification: - roles_mapping: + mappings: kubernetes_master: - kubernetes-master - helm diff --git a/schema/common/defaults/configuration/feature-mapping.yml b/schema/common/defaults/configuration/feature-mappings.yml similarity index 57% rename from schema/common/defaults/configuration/feature-mapping.yml rename to schema/common/defaults/configuration/feature-mappings.yml index bf8ba2c452..df1b5a816e 100644 --- a/schema/common/defaults/configuration/feature-mapping.yml +++ b/schema/common/defaults/configuration/feature-mappings.yml @@ -1,59 +1,9 @@ --- -kind: configuration/feature-mapping -title: "Feature mapping to roles" +kind: configuration/feature-mappings +title: "Feature mapping to components" name: default specification: - available_roles: - - name: repository - enabled: true - - name: firewall - enabled: true - - name: image-registry - enabled: true - - name: kubernetes-master - enabled: true - - name: kubernetes-node - enabled: true - - name: helm - enabled: true - - name: logging - enabled: true - - name: opensearch - enabled: true - - name: elasticsearch-curator - enabled: true - - name: opensearch-dashboards - enabled: true - - name: filebeat - enabled: true - - name: prometheus - enabled: true - - name: grafana - enabled: true - - name: node-exporter - enabled: true - - name: jmx-exporter - enabled: true - - name: zookeeper - enabled: true - - name: kafka - enabled: true - - name: rabbitmq - enabled: true - - name: kafka-exporter - enabled: true - - name: postgresql - enabled: true - - name: postgres-exporter - enabled: true - - name: haproxy - enabled: true - - name: applications - enabled: true - - name: rook - enabled: true - - roles_mapping: + mappings: kafka: - zookeeper - jmx-exporter diff --git a/schema/common/defaults/configuration/features.yml b/schema/common/defaults/configuration/features.yml new file mode 100644 index 0000000000..091223414c --- /dev/null +++ b/schema/common/defaults/configuration/features.yml @@ -0,0 +1,54 @@ +--- +kind: configuration/features +title: "Features to be enabled/disabled" +name: default +specification: + features: + - name: repository + enabled: true + - name: firewall + enabled: true + - name: image-registry + enabled: true + - name: kubernetes-master + enabled: true + - name: kubernetes-node + enabled: true + - name: helm + enabled: true + - name: logging + enabled: true + - name: opensearch + enabled: true + - name: elasticsearch-curator + enabled: true + - name: opensearch-dashboards + enabled: true + - name: filebeat + enabled: true + - name: prometheus + enabled: true + - name: grafana + enabled: true + - name: node-exporter + enabled: true + - name: jmx-exporter + enabled: true + - name: zookeeper + enabled: true + - name: kafka + enabled: true + - name: rabbitmq + enabled: true + - name: kafka-exporter + enabled: true + - name: postgresql + enabled: true + - name: postgres-exporter + enabled: true + - name: haproxy + enabled: true + - name: applications + enabled: true + - name: rook + enabled: true diff --git a/schema/common/defaults/configuration/image-registry.yml b/schema/common/defaults/configuration/image-registry.yml index 788ea12d06..f10c618aed 100644 --- a/schema/common/defaults/configuration/image-registry.yml +++ b/schema/common/defaults/configuration/image-registry.yml @@ -10,242 +10,251 @@ specification: images_to_load: x86_64: generic: - - name: "epiphanyplatform/keycloak:14.0.0" - file_name: keycloak-14.0.0.tar - - name: "rabbitmq:3.8.9" - file_name: rabbitmq-3.8.9.tar - - name: "kubernetesui/dashboard:v2.3.1" - file_name: dashboard-v2.3.1.tar - - name: "kubernetesui/metrics-scraper:v1.0.7" - file_name: metrics-scraper-v1.0.7.tar - # postgres - - name: bitnami/pgpool:4.2.4 - file_name: pgpool-4.2.4.tar - - name: bitnami/pgbouncer:1.16.0 - file_name: pgbouncer-1.16.0.tar - # ceph - - name: "rook/ceph:v1.8.8" - file_name: ceph-v1.8.8.tar + applications: + - name: "epiphanyplatform/keycloak:14.0.0" + file_name: keycloak-14.0.0.tar + - name: "rabbitmq:3.8.9" + file_name: rabbitmq-3.8.9.tar + - name: "bitnami/pgpool:4.2.4" + file_name: pgpool-4.2.4.tar + - name: "bitnami/pgbouncer:1.16.0" + file_name: pgbouncer-1.16.0.tar + kubernetes-master: + - name: "kubernetesui/dashboard:v2.3.1" + file_name: dashboard-v2.3.1.tar + - name: "kubernetesui/metrics-scraper:v1.0.7" + file_name: metrics-scraper-v1.0.7.tar + rook: + - name: "rook/ceph:v1.8.8" + file_name: ceph-v1.8.8.tar current: - - name: "haproxy:2.2.2-alpine" - file_name: haproxy-2.2.2-alpine.tar - # K8s v1.22.4 - Epiphany 1.3 - # https://github.com/kubernetes/kubernetes/blob/v1.22.4/build/dependencies.yaml - - name: "k8s.gcr.io/kube-apiserver:v1.22.4" - file_name: kube-apiserver-v1.22.4.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.22.4" - file_name: kube-controller-manager-v1.22.4.tar - - name: "k8s.gcr.io/kube-proxy:v1.22.4" - file_name: kube-proxy-v1.22.4.tar - - name: "k8s.gcr.io/kube-scheduler:v1.22.4" - file_name: kube-scheduler-v1.22.4.tar - - name: "k8s.gcr.io/coredns/coredns:v1.8.4" - file_name: coredns-v1.8.4.tar - - name: "k8s.gcr.io/etcd:3.5.0-0" - file_name: etcd-3.5.0-0.tar - - name: "k8s.gcr.io/pause:3.5" - file_name: pause-3.5.tar - - name: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0" - file_name: csi-attacher-v3.4.0.tar - - name: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0" - file_name: csi-node-driver-registrar-v2.5.0.tar - - name: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0" - file_name: csi-provisioner-v3.1.0.tar - - name: "k8s.gcr.io/sig-storage/csi-resizer:v1.4.0" - file_name: csi-resizer-v1.4.0.tar - - name: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1" - file_name: csi-snapshotter-v5.0.1.tar - # flannel - - name: "quay.io/coreos/flannel:v0.14.0-amd64" - file_name: flannel-v0.14.0-amd64.tar - - name: "quay.io/coreos/flannel:v0.14.0" - file_name: flannel-v0.14.0.tar - - name: "quay.io/ceph/ceph:v16.2.7" - file_name: ceph-v16.2.7.tar - - name: "quay.io/cephcsi/cephcsi:v3.5.1" - file_name: cephcsi-v3.5.1.tar - - name: "quay.io/csiaddons/k8s-sidecar:v0.2.1" - file_name: k8s-sidecar-v0.2.1.tar - - name: "quay.io/csiaddons/volumereplication-operator:v0.3.0" - file_name: volumereplication-operator-v0.3.0.tar - # canal & calico - - name: "calico/cni:v3.20.3" - file_name: cni-v3.20.3.tar - - name: "calico/kube-controllers:v3.20.3" - file_name: kube-controllers-v3.20.3.tar - - name: "calico/node:v3.20.3" - file_name: node-v3.20.3.tar - - name: "calico/pod2daemon-flexvol:v3.20.3" - file_name: pod2daemon-flexvol-v3.20.3.tar + haproxy: + - name: "haproxy:2.2.2-alpine" + file_name: haproxy-2.2.2-alpine.tar + kubernetes-master: + # K8s v1.22.4 - Epiphany 1.3 + # https://github.com/kubernetes/kubernetes/blob/v1.22.4/build/dependencies.yaml + - name: "k8s.gcr.io/kube-apiserver:v1.22.4" + file_name: kube-apiserver-v1.22.4.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.22.4" + file_name: kube-controller-manager-v1.22.4.tar + - name: "k8s.gcr.io/kube-proxy:v1.22.4" + file_name: kube-proxy-v1.22.4.tar + - name: "k8s.gcr.io/kube-scheduler:v1.22.4" + file_name: kube-scheduler-v1.22.4.tar + - name: "k8s.gcr.io/coredns/coredns:v1.8.4" + file_name: coredns-v1.8.4.tar + - name: "k8s.gcr.io/etcd:3.5.0-0" + file_name: etcd-3.5.0-0.tar + - name: "k8s.gcr.io/pause:3.5" + file_name: pause-3.5.tar + - name: "k8s.gcr.io/sig-storage/csi-attacher:v3.4.0" + file_name: csi-attacher-v3.4.0.tar + - name: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0" + file_name: csi-node-driver-registrar-v2.5.0.tar + - name: "k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0" + file_name: csi-provisioner-v3.1.0.tar + - name: "k8s.gcr.io/sig-storage/csi-resizer:v1.4.0" + file_name: csi-resizer-v1.4.0.tar + - name: "k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1" + file_name: csi-snapshotter-v5.0.1.tar + # flannel + - name: "quay.io/coreos/flannel:v0.14.0-amd64" + file_name: flannel-v0.14.0-amd64.tar + - name: "quay.io/coreos/flannel:v0.14.0" + file_name: flannel-v0.14.0.tar + - name: "quay.io/ceph/ceph:v16.2.7" + file_name: ceph-v16.2.7.tar + - name: "quay.io/cephcsi/cephcsi:v3.5.1" + file_name: cephcsi-v3.5.1.tar + - name: "quay.io/csiaddons/k8s-sidecar:v0.2.1" + file_name: k8s-sidecar-v0.2.1.tar + - name: "quay.io/csiaddons/volumereplication-operator:v0.3.0" + file_name: volumereplication-operator-v0.3.0.tar + # canal & calico + - name: "calico/cni:v3.20.3" + file_name: cni-v3.20.3.tar + - name: "calico/kube-controllers:v3.20.3" + file_name: kube-controllers-v3.20.3.tar + - name: "calico/node:v3.20.3" + file_name: node-v3.20.3.tar + - name: "calico/pod2daemon-flexvol:v3.20.3" + file_name: pod2daemon-flexvol-v3.20.3.tar legacy: - # K8s v1.21.7 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.21.7" - file_name: kube-apiserver-v1.21.7.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.21.7" - file_name: kube-controller-manager-v1.21.7.tar - - name: "k8s.gcr.io/kube-proxy:v1.21.7" - file_name: kube-proxy-v1.21.7.tar - - name: "k8s.gcr.io/kube-scheduler:v1.21.7" - file_name: kube-scheduler-v1.21.7.tar - - name: "k8s.gcr.io/coredns/coredns:v1.8.0" - file_name: coredns-v1.8.0.tar - - name: "k8s.gcr.io/etcd:3.4.13-0" - file_name: etcd-3.4.13-0.tar - - name: "k8s.gcr.io/pause:3.4.1" - file_name: pause-3.4.1.tar - # K8s v1.20.12 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.20.12" - file_name: kube-apiserver-v1.20.12.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.20.12" - file_name: kube-controller-manager-v1.20.12.tar - - name: "k8s.gcr.io/kube-proxy:v1.20.12" - file_name: kube-proxy-v1.20.12.tar - - name: "k8s.gcr.io/kube-scheduler:v1.20.12" - file_name: kube-scheduler-v1.20.12.tar - - name: "k8s.gcr.io/coredns:1.7.0" - file_name: coredns-1.7.0.tar - - name: "k8s.gcr.io/pause:3.2" - file_name: pause-3.2.tar - # K8s v1.19.15 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.19.15" - file_name: kube-apiserver-v1.19.15.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" - file_name: kube-controller-manager-v1.19.15.tar - - name: "k8s.gcr.io/kube-proxy:v1.19.15" - file_name: kube-proxy-v1.19.15.tar - - name: "k8s.gcr.io/kube-scheduler:v1.19.15" - file_name: kube-scheduler-v1.19.15.tar - # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 - - name: "k8s.gcr.io/kube-apiserver:v1.18.6" - file_name: kube-apiserver-v1.18.6.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" - file_name: kube-controller-manager-v1.18.6.tar - - name: "k8s.gcr.io/kube-proxy:v1.18.6" - file_name: kube-proxy-v1.18.6.tar - - name: "k8s.gcr.io/kube-scheduler:v1.18.6" - file_name: kube-scheduler-v1.18.6.tar - - name: "k8s.gcr.io/coredns:1.6.7" - file_name: coredns-1.6.7.tar - - name: "k8s.gcr.io/etcd:3.4.3-0" - file_name: etcd-3.4.3-0.tar - # flannel - - name: "quay.io/coreos/flannel:v0.12.0-amd64" - file_name: flannel-v0.12.0-amd64.tar - - name: "quay.io/coreos/flannel:v0.12.0" - file_name: flannel-v0.12.0.tar - # canal & calico - - name: "calico/cni:v3.15.0" - file_name: cni-v3.15.0.tar - - name: "calico/kube-controllers:v3.15.0" - file_name: kube-controllers-v3.15.0.tar - - name: "calico/node:v3.15.0" - file_name: node-v3.15.0.tar - - name: "calico/pod2daemon-flexvol:v3.15.0" - file_name: pod2daemon-flexvol-v3.15.0.tar + kubernetes-master: + # K8s v1.21.7 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.21.7" + file_name: kube-apiserver-v1.21.7.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.21.7" + file_name: kube-controller-manager-v1.21.7.tar + - name: "k8s.gcr.io/kube-proxy:v1.21.7" + file_name: kube-proxy-v1.21.7.tar + - name: "k8s.gcr.io/kube-scheduler:v1.21.7" + file_name: kube-scheduler-v1.21.7.tar + - name: "k8s.gcr.io/coredns/coredns:v1.8.0" + file_name: coredns-v1.8.0.tar + - name: "k8s.gcr.io/etcd:3.4.13-0" + file_name: etcd-3.4.13-0.tar + - name: "k8s.gcr.io/pause:3.4.1" + file_name: pause-3.4.1.tar + # K8s v1.20.12 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.20.12" + file_name: kube-apiserver-v1.20.12.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.20.12" + file_name: kube-controller-manager-v1.20.12.tar + - name: "k8s.gcr.io/kube-proxy:v1.20.12" + file_name: kube-proxy-v1.20.12.tar + - name: "k8s.gcr.io/kube-scheduler:v1.20.12" + file_name: kube-scheduler-v1.20.12.tar + - name: "k8s.gcr.io/coredns:1.7.0" + file_name: coredns-1.7.0.tar + - name: "k8s.gcr.io/pause:3.2" + file_name: pause-3.2.tar + # K8s v1.19.15 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.19.15" + file_name: kube-apiserver-v1.19.15.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" + file_name: kube-controller-manager-v1.19.15.tar + - name: "k8s.gcr.io/kube-proxy:v1.19.15" + file_name: kube-proxy-v1.19.15.tar + - name: "k8s.gcr.io/kube-scheduler:v1.19.15" + file_name: kube-scheduler-v1.19.15.tar + # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 + - name: "k8s.gcr.io/kube-apiserver:v1.18.6" + file_name: kube-apiserver-v1.18.6.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" + file_name: kube-controller-manager-v1.18.6.tar + - name: "k8s.gcr.io/kube-proxy:v1.18.6" + file_name: kube-proxy-v1.18.6.tar + - name: "k8s.gcr.io/kube-scheduler:v1.18.6" + file_name: kube-scheduler-v1.18.6.tar + - name: "k8s.gcr.io/coredns:1.6.7" + file_name: coredns-1.6.7.tar + - name: "k8s.gcr.io/etcd:3.4.3-0" + file_name: etcd-3.4.3-0.tar + # flannel + - name: "quay.io/coreos/flannel:v0.12.0-amd64" + file_name: flannel-v0.12.0-amd64.tar + - name: "quay.io/coreos/flannel:v0.12.0" + file_name: flannel-v0.12.0.tar + # canal & calico + - name: "calico/cni:v3.15.0" + file_name: cni-v3.15.0.tar + - name: "calico/kube-controllers:v3.15.0" + file_name: kube-controllers-v3.15.0.tar + - name: "calico/node:v3.15.0" + file_name: node-v3.15.0.tar + - name: "calico/pod2daemon-flexvol:v3.15.0" + file_name: pod2daemon-flexvol-v3.15.0.tar aarch64: generic: - - name: "epiphanyplatform/keycloak:14.0.0" - file_name: keycloak-14.0.0.tar - - name: "rabbitmq:3.8.9" - file_name: rabbitmq-3.8.9.tar - - name: "kubernetesui/dashboard:v2.3.1" - file_name: dashboard-v2.3.1.tar - - name: "kubernetesui/metrics-scraper:v1.0.7" - file_name: metrics-scraper-v1.0.7.tar + applications: + - name: "epiphanyplatform/keycloak:14.0.0" + file_name: keycloak-14.0.0.tar + - name: "rabbitmq:3.8.9" + file_name: rabbitmq-3.8.9.tar + kubernetes-master: + - name: "kubernetesui/dashboard:v2.3.1" + file_name: dashboard-v2.3.1.tar + - name: "kubernetesui/metrics-scraper:v1.0.7" + file_name: metrics-scraper-v1.0.7.tar current: - - name: "haproxy:2.2.2-alpine" - file_name: haproxy-2.2.2-alpine.tar - # K8s v1.21.7 - Epiphany 1.3 - - name: "k8s.gcr.io/kube-apiserver:v1.22.4" - file_name: kube-apiserver-v1.22.4.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.22.4" - file_name: kube-controller-manager-v1.22.4.tar - - name: "k8s.gcr.io/kube-proxy:v1.22.4" - file_name: kube-proxy-v1.22.4.tar - - name: "k8s.gcr.io/kube-scheduler:v1.22.4" - file_name: kube-scheduler-v1.22.4.tar - - name: "k8s.gcr.io/coredns/coredns:v1.8.4" - file_name: coredns-v1.8.4.tar - - name: "k8s.gcr.io/etcd:3.5.0-0" - file_name: etcd-3.5.0-0.tar - - name: "k8s.gcr.io/pause:3.5" - file_name: pause-3.5.tar - # flannel - - name: "quay.io/coreos/flannel:v0.14.0-arm64" - file_name: flannel-v0.14.0-arm64.tar - - name: "quay.io/coreos/flannel:v0.14.0" - file_name: flannel-v0.14.0.tar - # canal & calico - - name: "calico/cni:v3.20.3" - file_name: cni-v3.20.3.tar - - name: "calico/kube-controllers:v3.20.3" - file_name: kube-controllers-v3.20.3.tar - - name: "calico/node:v3.20.3" - file_name: node-v3.20.3.tar - - name: "calico/pod2daemon-flexvol:v3.20.3" - file_name: pod2daemon-flexvol-v3.20.3.tar + haproxy: + - name: "haproxy:2.2.2-alpine" + file_name: haproxy-2.2.2-alpine.tar + kubernetes-master: + # K8s v1.21.7 - Epiphany 1.3 + - name: "k8s.gcr.io/kube-apiserver:v1.22.4" + file_name: kube-apiserver-v1.22.4.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.22.4" + file_name: kube-controller-manager-v1.22.4.tar + - name: "k8s.gcr.io/kube-proxy:v1.22.4" + file_name: kube-proxy-v1.22.4.tar + - name: "k8s.gcr.io/kube-scheduler:v1.22.4" + file_name: kube-scheduler-v1.22.4.tar + - name: "k8s.gcr.io/coredns/coredns:v1.8.4" + file_name: coredns-v1.8.4.tar + - name: "k8s.gcr.io/etcd:3.5.0-0" + file_name: etcd-3.5.0-0.tar + - name: "k8s.gcr.io/pause:3.5" + file_name: pause-3.5.tar + # flannel + - name: "quay.io/coreos/flannel:v0.14.0-arm64" + file_name: flannel-v0.14.0-arm64.tar + - name: "quay.io/coreos/flannel:v0.14.0" + file_name: flannel-v0.14.0.tar + # canal & calico + - name: "calico/cni:v3.20.3" + file_name: cni-v3.20.3.tar + - name: "calico/kube-controllers:v3.20.3" + file_name: kube-controllers-v3.20.3.tar + - name: "calico/node:v3.20.3" + file_name: node-v3.20.3.tar + - name: "calico/pod2daemon-flexvol:v3.20.3" + file_name: pod2daemon-flexvol-v3.20.3.tar legacy: - # K8s v1.21.7 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.21.7" - file_name: kube-apiserver-v1.21.7.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.21.7" - file_name: kube-controller-manager-v1.21.7.tar - - name: "k8s.gcr.io/kube-proxy:v1.21.7" - file_name: kube-proxy-v1.21.7.tar - - name: "k8s.gcr.io/kube-scheduler:v1.21.7" - file_name: kube-scheduler-v1.21.7.tar - - name: "k8s.gcr.io/coredns/coredns:v1.8.0" - file_name: coredns-v1.8.0.tar - - name: "k8s.gcr.io/etcd:3.4.13-0" - file_name: etcd-3.4.13-0.tar - - name: "k8s.gcr.io/pause:3.4.1" - file_name: pause-3.4.1.tar - # K8s v1.20.12 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.20.12" - file_name: kube-apiserver-v1.20.12.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.20.12" - file_name: kube-controller-manager-v1.20.12.tar - - name: "k8s.gcr.io/kube-proxy:v1.20.12" - file_name: kube-proxy-v1.20.12.tar - - name: "k8s.gcr.io/kube-scheduler:v1.20.12" - file_name: kube-scheduler-v1.20.12.tar - - name: "k8s.gcr.io/coredns:1.7.0" - file_name: coredns-1.7.0.tar - - name: "k8s.gcr.io/pause:3.2" - file_name: pause-3.2.tar - # K8s v1.19.15 - Epiphany 1.3 (transitional version) - - name: "k8s.gcr.io/kube-apiserver:v1.19.15" - file_name: kube-apiserver-v1.19.15.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" - file_name: kube-controller-manager-v1.19.15.tar - - name: "k8s.gcr.io/kube-proxy:v1.19.15" - file_name: kube-proxy-v1.19.15.tar - - name: "k8s.gcr.io/kube-scheduler:v1.19.15" - file_name: kube-scheduler-v1.19.15.tar - # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 - - name: "k8s.gcr.io/kube-apiserver:v1.18.6" - file_name: kube-apiserver-v1.18.6.tar - - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" - file_name: kube-controller-manager-v1.18.6.tar - - name: "k8s.gcr.io/kube-proxy:v1.18.6" - file_name: kube-proxy-v1.18.6.tar - - name: "k8s.gcr.io/kube-scheduler:v1.18.6" - file_name: kube-scheduler-v1.18.6.tar - - name: "k8s.gcr.io/coredns:1.6.7" - file_name: coredns-1.6.7.tar - - name: "k8s.gcr.io/etcd:3.4.3-0" - file_name: etcd-3.4.3-0.tar - # flannel - - name: "quay.io/coreos/flannel:v0.12.0-arm64" - file_name: flannel-v0.12.0-arm64.tar - - name: "quay.io/coreos/flannel:v0.12.0" - file_name: flannel-v0.12.0.tar - # canal & calico - - name: "calico/cni:v3.15.0" - file_name: cni-v3.15.0.tar - - name: "calico/kube-controllers:v3.15.0" - file_name: kube-controllers-v3.15.0.tar - - name: "calico/node:v3.15.0" - file_name: node-v3.15.0.tar - - name: "calico/pod2daemon-flexvol:v3.15.0" - file_name: pod2daemon-flexvol-v3.15.0.tar + kubernetes-master: + # K8s v1.21.7 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.21.7" + file_name: kube-apiserver-v1.21.7.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.21.7" + file_name: kube-controller-manager-v1.21.7.tar + - name: "k8s.gcr.io/kube-proxy:v1.21.7" + file_name: kube-proxy-v1.21.7.tar + - name: "k8s.gcr.io/kube-scheduler:v1.21.7" + file_name: kube-scheduler-v1.21.7.tar + - name: "k8s.gcr.io/coredns/coredns:v1.8.0" + file_name: coredns-v1.8.0.tar + - name: "k8s.gcr.io/etcd:3.4.13-0" + file_name: etcd-3.4.13-0.tar + - name: "k8s.gcr.io/pause:3.4.1" + file_name: pause-3.4.1.tar + # K8s v1.20.12 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.20.12" + file_name: kube-apiserver-v1.20.12.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.20.12" + file_name: kube-controller-manager-v1.20.12.tar + - name: "k8s.gcr.io/kube-proxy:v1.20.12" + file_name: kube-proxy-v1.20.12.tar + - name: "k8s.gcr.io/kube-scheduler:v1.20.12" + file_name: kube-scheduler-v1.20.12.tar + - name: "k8s.gcr.io/coredns:1.7.0" + file_name: coredns-1.7.0.tar + - name: "k8s.gcr.io/pause:3.2" + file_name: pause-3.2.tar + # K8s v1.19.15 - Epiphany 1.3 (transitional version) + - name: "k8s.gcr.io/kube-apiserver:v1.19.15" + file_name: kube-apiserver-v1.19.15.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.19.15" + file_name: kube-controller-manager-v1.19.15.tar + - name: "k8s.gcr.io/kube-proxy:v1.19.15" + file_name: kube-proxy-v1.19.15.tar + - name: "k8s.gcr.io/kube-scheduler:v1.19.15" + file_name: kube-scheduler-v1.19.15.tar + # K8s v1.18.6 - Epiphany 0.7.1 - 1.2 + - name: "k8s.gcr.io/kube-apiserver:v1.18.6" + file_name: kube-apiserver-v1.18.6.tar + - name: "k8s.gcr.io/kube-controller-manager:v1.18.6" + file_name: kube-controller-manager-v1.18.6.tar + - name: "k8s.gcr.io/kube-proxy:v1.18.6" + file_name: kube-proxy-v1.18.6.tar + - name: "k8s.gcr.io/kube-scheduler:v1.18.6" + file_name: kube-scheduler-v1.18.6.tar + - name: "k8s.gcr.io/coredns:1.6.7" + file_name: coredns-1.6.7.tar + - name: "k8s.gcr.io/etcd:3.4.3-0" + file_name: etcd-3.4.3-0.tar + # flannel + - name: "quay.io/coreos/flannel:v0.12.0-arm64" + file_name: flannel-v0.12.0-arm64.tar + - name: "quay.io/coreos/flannel:v0.12.0" + file_name: flannel-v0.12.0.tar + # canal & calico + - name: "calico/cni:v3.15.0" + file_name: cni-v3.15.0.tar + - name: "calico/kube-controllers:v3.15.0" + file_name: kube-controllers-v3.15.0.tar + - name: "calico/node:v3.15.0" + file_name: node-v3.15.0.tar + - name: "calico/pod2daemon-flexvol:v3.15.0" + file_name: pod2daemon-flexvol-v3.15.0.tar diff --git a/schema/common/validation/configuration/feature-mapping.yml b/schema/common/validation/configuration/feature-mappings.yml similarity index 78% rename from schema/common/validation/configuration/feature-mapping.yml rename to schema/common/validation/configuration/feature-mappings.yml index 92bc0c699c..a1d919e4a8 100644 --- a/schema/common/validation/configuration/feature-mapping.yml +++ b/schema/common/validation/configuration/feature-mappings.yml @@ -1,19 +1,10 @@ --- "$id": "#/specification" -title: "Feature-mapping specification schema" -description: "Feature-mapping specification schema" +title: "Feature-mappings specification schema" +description: "Feature-mappings specification schema" type: object properties: - available_roles: - type: array - items: - type: object - properties: - name: - type: string - enabled: - type: boolean - roles_mapping: + mappings: type: object properties: kafka: diff --git a/schema/common/validation/configuration/features.yml b/schema/common/validation/configuration/features.yml new file mode 100644 index 0000000000..91df885b09 --- /dev/null +++ b/schema/common/validation/configuration/features.yml @@ -0,0 +1,15 @@ +--- +"$id": "#/specification" +title: "Features to be enabled/disabled schema" +description: "Features to be enabled/disabled schema" +type: object +properties: + features: + type: array + items: + type: object + properties: + name: + type: string + enabled: + type: boolean diff --git a/schema/common/validation/configuration/image-registry.yml b/schema/common/validation/configuration/image-registry.yml index 8bd0ad3173..a08d12c701 100644 --- a/schema/common/validation/configuration/image-registry.yml +++ b/schema/common/validation/configuration/image-registry.yml @@ -20,54 +20,72 @@ properties: type: object properties: generic: - type: array - items: - type: object - properties: - name: - type: string - file_name: - type: string + type: object + properties: + [applications, rabbitmq, kubernetes-master, postgresql, rook]: + type: array + items: + type: object + properties: + name: + type: string + file_name: + type: string current: - type: array - items: - type: object - properties: - name: - type: string - file_name: - type: string + type: object + properties: + [haproxy, kubernetes-master]: + type: array + items: + type: object + properties: + name: + type: string + file_name: + type: string legacy: - type: array - items: - type: object - properties: - name: - type: string - file_name: - type: string + type: object + properties: + kubernetes-master: + type: array + items: + type: object + properties: + name: + type: string + file_name: + type: string aarch64: type: object properties: generic: - type: array - items: - type: object - properties: - name: - type: string - file_name: - type: string + type: object + properties: + [applications, rabbitmq, kubernetes-master]: + type: array + items: + type: object + properties: + name: + type: string + file_name: + type: string current: - type: array - items: - type: object - properties: - name: - type: string - file_name: - type: string + type: object + properties: + [haproxy, kubernetes-master]: + type: array + items: + type: object + properties: + name: + type: string + file_name: + type: string legacy: - type: array - items: - items: {} + type: object + properties: + kubernetes-master: + type: array + items: + items: {} diff --git a/schema/common/validation/epiphany-cluster.yml b/schema/common/validation/epiphany-cluster.yml index cc1afed541..793bf8c76e 100644 --- a/schema/common/validation/epiphany-cluster.yml +++ b/schema/common/validation/epiphany-cluster.yml @@ -144,9 +144,9 @@ properties: properties: kubernetes_master: properties: - count: { type: integer, enum: [0] } + count: {type: integer, enum: [0]} then: properties: kubernetes_node: properties: - count: { type: integer, enum: [0] } + count: {type: integer, enum: [0]}