[2.0.1] Adaptive mode for downloading requirements (#3188)

* Split available_roles and roles_mapping into separate yaml documents (#3097) (#3119) * available_roles splitted into feature-mappings and features documents * feature-mappings added to the Init by default * Add manifest file parsing (#3105) (#3130) * Add `-m/--manifest` flag to accept manifest.yml produced by `epicli init/prepare` * Add `-v/--verbose` mode for printing out parsed manifest data * Add ManifestReader class used for paring the manifest.yml file * Move src/command/*.py to debian/redhat subdirs where needed * Optimize Grafana dashboards downloading (#3131) (#3150) * Optimize files downloading (#3116) (#3156) * Add image-registry configuration reading (#3106) (#3159)
hitachienergy · Jun 28, 2022 · 0229549 · 0229549
1 parent 05eae80
commit 0229549
Show file tree

Hide file tree

Showing 67 changed files with 2,717 additions and 717 deletions.
diff --git a/.ansible-lint b/.ansible-lint
@@ -38,6 +38,7 @@ skip_list:
   - meta-no-info
   - package-latest
   - fqcn-builtins
+  - no-jinja-when
 
 ##################
 # Tags to follow #

diff --git a/ansible/playbooks/filter_plugins/container.py b/ansible/playbooks/filter_plugins/container.py
@@ -0,0 +1,27 @@
+from typing import Any, Dict, List
+
+
+class FilterModule:
+    """ Filters for Python's container types """
+
+    def filters(self):
+        return {
+            'dict_to_list': self.dict_to_list
+        }
+
+    def dict_to_list(self, data: Dict, only_values: bool = False, only_keys: bool = False) -> List:
+        """
+        Convert dict to list without using Ansible's loop mechanism with dict2items filter.
+
+        :param data: to be converted into a list
+        :param only_values: construct list with only dict's values
+        :param only_keys: construct list with only dict's keys
+        :return: data transformed into a list
+        """
+        if only_values:
+            return list(data.values())
+
+        if only_keys:
+            return list(data.keys())
+
+        return list(data.items())
diff --git a/ansible/playbooks/repository.yml b/ansible/playbooks/repository.yml
@@ -1,5 +1,5 @@
 ---
-# This playbook is empty by purpose, just to enable repository role in configuration/feature-mapping
+# This playbook is empty by purpose, just to enable repository role in configuration/features
 # to populate defaults/configuration to Ansible vars
 - hosts: "!all"
   tasks: []
diff --git a/ansible/playbooks/roles/download/tasks/list_files.yml b/ansible/playbooks/roles/download/tasks/list_files.yml
@@ -1,25 +1,9 @@
 ---
-- name: Get file listing
-  uri:
-    method: GET
-    url: "{{ repository_url }}/files/?F=0"  # F=0 formats the listing as a simple list (not FancyIndexed)
-    body_format: raw
-    return_content: true
-    validate_certs: "{{ validate_certs | default(false, true) | bool }}"  # handling "undefined", "null", "empty" and "boolean" values all at once
-  register: uri_list_files
-  until: uri_list_files is success
-  retries: 3
-  delay: 2
-  become: false
+- name: Get files list from the repository
+  include_tasks: list_requirements.yml
+  vars:
+    _requirements: files
 
-# TODO: make it work with yaml or json (instead of html, sic!).
-- name: Parse html response and return file listing
+- name: Set files in repository as fact
   set_fact:
-    list_files_result: >-
-      {{ lines | select('match', regexp)
-               | reject('match', '.*Parent Directory.*')
-               | map('regex_replace', regexp, '\1')
-               | list }}
-  vars:
-    lines: "{{ uri_list_files.content.splitlines() }}"
-    regexp: '.*<li><a href="([^"]+)".*'
+    list_files_result: "{{ list_requirements_result }}"
diff --git a/ansible/playbooks/roles/download/tasks/list_images.yml b/ansible/playbooks/roles/download/tasks/list_images.yml
@@ -0,0 +1,9 @@
+---
+- name: Get images list from the repository
+  include_tasks: list_requirements.yml
+  vars:
+    _requirements: images
+
+- name: Set images in repository as fact
+  set_fact:
+    list_images_result: "{{ list_requirements_result }}"
diff --git a/ansible/playbooks/roles/download/tasks/list_requirements.yml b/ansible/playbooks/roles/download/tasks/list_requirements.yml
@@ -0,0 +1,25 @@
+---
+- name: Get requirements listing
+  uri:
+    method: GET
+    url: "{{ repository_url }}/{{ _requirements }}/?F=0"  # F=0 formats the listing as a simple list (not FancyIndexed)
+    body_format: raw
+    return_content: true
+    validate_certs: "{{ validate_certs | default(false, true) | bool }}"  # handling "undefined", "null", "empty" and "boolean" values all at once
+  register: uri_list_files
+  until: uri_list_files is success
+  retries: 3
+  delay: 2
+  become: false
+
+# TODO: make it work with yaml or json (instead of html, sic!).
+- name: Parse html response and return requirements listing
+  set_fact:
+    list_requirements_result: >-
+      {{ lines | select('match', regexp)
+               | reject('match', '.*Parent Directory.*')
+               | map('regex_replace', regexp, '\1')
+               | list }}
+  vars:
+    lines: "{{ uri_list_files.content.splitlines() }}"
+    regexp: '.*<li><a href="([^"]+)".*'
diff --git a/ansible/playbooks/roles/image_registry/tasks/main.yml b/ansible/playbooks/roles/image_registry/tasks/main.yml
@@ -122,24 +122,59 @@
             --name {{ epiphany_registry.container_name }} -v {{ epiphany_registry.volume_name }}:/var/lib/registry
             {{ specification.registry_image.name }}
 
-    - name: Set images to load
+    - name: Define images to unpack
       set_fact:
-        generic_and_current_images: >-
-          {{ specification.images_to_load[ansible_architecture].generic + specification.images_to_load[ansible_architecture].current }}
-        legacy_images: "{{ specification.images_to_load[ansible_architecture].legacy }}"
+        current_schema_images: "{{ specification.images_to_load[ansible_architecture].current }}"
+        generic_schema_images: "{{ specification.images_to_load[ansible_architecture].generic }}"
+        legacy_schema_images: "{{ specification.images_to_load[ansible_architecture].legacy }}"
+
+    - name: Initialize image facts
+      set_fact:
+        requested_images: []
+        current_images: []
+        generic_images: []
+        legacy_images: []
+
+    - name: Set list of current images to be loaded/pushed
+      set_fact:
+        current_images: "{{ current_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Set list of generic images to be loaded/pushed
+      set_fact:
+        generic_images: "{{ generic_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Set list of legacy images to be loaded/pushed
+      set_fact:
+        legacy_images: "{{ legacy_schema_images | dict_to_list(only_values='True') | flatten }}"
+
+    - name: Merge current and generic images
+      set_fact:
+        current_and_generic_images: >-
+          {{ current_images + generic_images }}
+
+    - name: Get list of available images
+      include_role:
+        name: download
+        tasks_from: list_images.yml
+
+    - name: Filter only requested images
+      set_fact:  # gather only images listed in schema to avoid downloading unknown files
+        requested_images: "{{ requested_images + [item] }}"
+      when: "{{ item.file_name in list_images_result }}"
+      loop: "{{ current_and_generic_images }}"
 
     - name: Load generic and current version images
       vars:
         docker_image: "{{ item }}"
       include_tasks: load-image.yml
-      loop: "{{ generic_and_current_images }}"
+      loop: "{{ requested_images }}"
 
     - name: Push generic and current version images to registry
       vars:
         docker_image: "{{ item }}"
         new_image_tag: "{{ image_registry_address }}/{{ item.name }}"
       include_tasks: push-image.yml
-      loop: "{{ generic_and_current_images }}"
+      loop: "{{ requested_images }}"
 
     - name: Load legacy version images to registry when upgrading
       when: is_upgrade_run

diff --git a/ansible/playbooks/roles/opensearch/defaults/main.yml b/ansible/playbooks/roles/opensearch/defaults/main.yml
@@ -59,4 +59,3 @@ certificates:
 ports:
   http: 9200
   transport: 9300
-log4j_file_name: apache-log4j-2.17.1-bin.tar.gz
diff --git a/ansible/playbooks/roles/repository/defaults/main.yml b/ansible/playbooks/roles/repository/defaults/main.yml
@@ -1,4 +1,5 @@
 ---
 download_requirements_dir: "/var/tmp/epi-download-requirements"
-download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py"
 download_requirements_flag: "{{ download_requirements_dir }}/download-requirements-done.flag"
+download_requirements_manifest: "{{ download_requirements_dir }}/manifest.yml"
+download_requirements_script: "{{ download_requirements_dir }}/download-requirements.py"
diff --git a/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py b/ansible/playbooks/roles/repository/files/download-requirements/download-requirements.py
@@ -9,7 +9,7 @@
 
 from src.command.toolchain import TOOLCHAINS
 from src.config.config import Config
-from src.error import DownloadRequirementsError
+from src.error import DownloadRequirementsException
 
 
 def install_missing_modules(config: Config):
@@ -79,7 +79,7 @@ def main(argv: List[str]) -> int:
 
         time_end = datetime.datetime.now() - time_begin
         logging.info(f'Total execution time: {str(time_end).split(".")[0]}')
-    except DownloadRequirementsError:
+    except DownloadRequirementsException:
         return 1
 
     return 0

diff --git a/...ble/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml b/...ble/playbooks/roles/repository/files/download-requirements/requirements/aarch64/files.yml
@@ -3,45 +3,55 @@ files:
   # --- Exporters ---
   'https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-arm64.tar.gz':
     sha256: 95ff0c723f3cdb6967b54c0208a5d0e67ad59dc53c1907a401cb8a448e53ec96
+    deps: [kafka-exporter]
 
   'https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar':
     sha256: 0ddc6834f854c03d5795305193c1d33132a24fbd406b4b52828602f5bc30777e
+    deps: [kafka]
 
   'https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-arm64.tar.gz':
     sha256: f19f35175f87d41545fa7d4657e834e3a37c1fe69f3bf56bc031a256117764e7
+    deps: [node-exporter]
 
   'https://github.com/prometheus-community/postgres_exporter/releases/download/v0.10.0/postgres_exporter-0.10.0.linux-arm64.tar.gz':
     sha256: 82a1a4e07c7140f8e55532dbbdfea3bbba33dafc7ef0a221601bb2fd5359ff03
+    deps: [postgres-exporter]
 
   # --- Misc ---
   'https://archive.apache.org/dist/kafka/2.8.1/kafka_2.12-2.8.1.tgz':
     sha256: 175a4134efc569a586d58916cd16ce70f868b13dea2b5a3d12a67b1395d59f98
+    deps: [kafka]
 
   'https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz':
     sha256: c35ed6786d59b73920243f1a324d24c2ddfafb379041d7a350cc9a341c52caf3
+    deps: [zookeeper]
 
   'https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-arm64.tar.gz':
     sha256: afa44f350797032ceb714598900cfdddbf81d6ef03d2ecbfc0221cc2cb28a6b9
+    deps: [prometheus]
 
   'https://github.com/prometheus/prometheus/releases/download/v2.31.1/prometheus-2.31.1.linux-arm64.tar.gz':
     sha256: a7b4694b96cbf38b63ca92d05a6d3a2cf6df50a85a4d2a3fe2d758a65dcbec3b
+    deps: [prometheus]
 
   'https://get.helm.sh/helm-v3.2.0-linux-arm64.tar.gz':
     sha256: cd11f0ed12a658f3b78392528814350a508d2c53d8da7f04145909e94bda10f1
-
-  'https://archive.apache.org/dist/logging/log4j/2.17.1/apache-log4j-2.17.1-bin.tar.gz':
-    sha256: b876c20c9d318d77a39c0c2e095897b2bb1cd100c7859643f8c7c8b0fc6d5961
+    deps: [helm]
 
   # --- Helm charts ---
   'https://charts.bitnami.com/bitnami/node-exporter-2.3.17.tgz':
     sha256: ec586fabb775a4f05510386899cf348391523c89ff5a1d4097b0592e675ade7f
+    deps: [kubernetes-master]
 
   'https://helm.elastic.co/helm/filebeat/filebeat-7.12.1.tgz':
     sha256: 5838058fe06372390dc335900a7707109cc7287a84164ca245d395af1f9c0a79
+    deps: [kubernetes-master]
 
   # --- OpenSearch Bundle ---
   'https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.4/opensearch-1.2.4-linux-arm64.tar.gz':
     sha256: 5e8cd13ad1831e4a286a54334505c16c43ce8e50981100eea4eb18f79d3e63a5
+    deps: [logging, opensearch]
 
   'https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/1.2.0/opensearch-dashboards-1.2.0-linux-arm64.tar.gz':
     sha256: 1f668d98f4670f1b88f03b19d30b2cc44ec439a7b2edff1a48034717d594cfe1
+    deps: [logging, opensearch]
diff --git a/...le/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml b/...le/playbooks/roles/repository/files/download-requirements/requirements/aarch64/images.yml
@@ -1,62 +1,64 @@
 ---
 images:
-  'haproxy:2.2.2-alpine':
-    sha1: 2fd3bd554ee6e126a1b1e5055ed0349beac81ffc
+  haproxy:
+    'haproxy:2.2.2-alpine':
+      sha1: 2fd3bd554ee6e126a1b1e5055ed0349beac81ffc
 
-  'kubernetesui/dashboard:v2.3.1':
-    sha1: 5dc90f59af4643952d5a728213983e6f3d884895
+  image-registry:
+    'registry:2.8.0':
+      sha1: c6cdd738fbdd2efaef9588b439fc3a7c0c090368
+      allow_mismatch: true
 
-  'kubernetesui/metrics-scraper:v1.0.7':
-    sha1: 90b57b399e7ed44bad422e7d9572bfd6d737724a
+  applications:
+    'epiphanyplatform/keycloak:14.0.0':
+      sha1: c73be9d38580fc9819bec8942e2d2569196547e1
 
-  'registry:2.8.0':
-    sha1: c6cdd738fbdd2efaef9588b439fc3a7c0c090368
-    allow_mismatch: true
+    'rabbitmq:3.8.9':
+      sha1: 854d06bae1ee7e3a94570e1ce104618430988f57
 
-  # applications
-  'epiphanyplatform/keycloak:14.0.0':
-    sha1: c73be9d38580fc9819bec8942e2d2569196547e1
+  kubernetes-master:
+    'kubernetesui/dashboard:v2.3.1':
+      sha1: 5dc90f59af4643952d5a728213983e6f3d884895
 
-  'rabbitmq:3.8.9':
-    sha1: 854d06bae1ee7e3a94570e1ce104618430988f57
+    'kubernetesui/metrics-scraper:v1.0.7':
+      sha1: 90b57b399e7ed44bad422e7d9572bfd6d737724a
+    # K8s
+    # v1.22.4
+    'k8s.gcr.io/kube-apiserver:v1.22.4':
+      sha1: 6e101cfa4384346b45701e6dda5591a41fa5776d
 
-  # K8s
-  # v1.22.4
-  'k8s.gcr.io/kube-apiserver:v1.22.4':
-    sha1: 6e101cfa4384346b45701e6dda5591a41fa5776d
+    'k8s.gcr.io/kube-controller-manager:v1.22.4':
+      sha1: 6561280956af24f9547dabde5758a4091558e771
 
-  'k8s.gcr.io/kube-controller-manager:v1.22.4':
-    sha1: 6561280956af24f9547dabde5758a4091558e771
+    'k8s.gcr.io/kube-scheduler:v1.22.4':
+      sha1: e224852d58ab649f3145cae3ed4f2926e66117bf
 
-  'k8s.gcr.io/kube-scheduler:v1.22.4':
-    sha1: e224852d58ab649f3145cae3ed4f2926e66117bf
+    'k8s.gcr.io/kube-proxy:v1.22.4':
+      sha1: 5e5e4032f3f6464ede1a4a85854013d0801c8eff
 
-  'k8s.gcr.io/kube-proxy:v1.22.4':
-    sha1: 5e5e4032f3f6464ede1a4a85854013d0801c8eff
+    'k8s.gcr.io/coredns/coredns:v1.8.4':
+      sha1: e5d7de1974e8f331892d9587a5312d4cdda04bb2
 
-  'k8s.gcr.io/coredns/coredns:v1.8.4':
-    sha1: e5d7de1974e8f331892d9587a5312d4cdda04bb2
+    'k8s.gcr.io/etcd:3.5.0-0':
+      sha1: fb1975ba3fc696fa7530c0752d15abe8ea23e80d
 
-  'k8s.gcr.io/etcd:3.5.0-0':
-    sha1: fb1975ba3fc696fa7530c0752d15abe8ea23e80d
+    'k8s.gcr.io/pause:3.5':
+      sha1: 8f3be7cc532c25b01a2e5e0943b4d55bce0b0f1c
 
-  'k8s.gcr.io/pause:3.5':
-    sha1: 8f3be7cc532c25b01a2e5e0943b4d55bce0b0f1c
+    'quay.io/coreos/flannel:v0.14.0-arm64':
+      sha1: 342b806ab5da6133438f925e22ced802e0e56861
 
-  'quay.io/coreos/flannel:v0.14.0-arm64':
-    sha1: 342b806ab5da6133438f925e22ced802e0e56861
+    'quay.io/coreos/flannel:v0.14.0':
+      sha1: 098ec78af9bf3a70afbd1a9743ff352f72fb036d
 
-  'quay.io/coreos/flannel:v0.14.0':
-    sha1: 098ec78af9bf3a70afbd1a9743ff352f72fb036d
+    'calico/cni:v3.20.3':
+      sha1: 5338d670e01558dd053768fb3cdc39c8568d6f4c
 
-  'calico/cni:v3.20.3':
-    sha1: 5338d670e01558dd053768fb3cdc39c8568d6f4c
+    'calico/kube-controllers:v3.20.3':
+      sha1: 24615f8f9c0cac651401cc633ba4850b8f17e175
 
-  'calico/kube-controllers:v3.20.3':
-    sha1: 24615f8f9c0cac651401cc633ba4850b8f17e175
+    'calico/node:v3.20.3':
+      sha1: c0432602b9e8dd47d3c869759f210305d9d87d90
 
-  'calico/node:v3.20.3':
-    sha1: c0432602b9e8dd47d3c869759f210305d9d87d90
-
-  'calico/pod2daemon-flexvol:v3.20.3':
-    sha1: 6e1e59ad1cf6af46d4ce8c8b16f4014337c22411
+    'calico/pod2daemon-flexvol:v3.20.3':
+      sha1: 6e1e59ad1cf6af46d4ce8c8b16f4014337c22411