diff --git a/VERSION b/VERSION index c921c13021..889dbd3fd2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0dev +2.0.1dev diff --git a/ansible/playbooks/roles/common/tasks/RedHat.yml b/ansible/playbooks/roles/common/tasks/RedHat.yml index 920e6e2f2a..783d19a16d 100644 --- a/ansible/playbooks/roles/common/tasks/RedHat.yml +++ b/ansible/playbooks/roles/common/tasks/RedHat.yml @@ -49,3 +49,7 @@ lineinfile: path: /home/{{ admin_user.name }}/.bash_profile line: source /etc/profile.d/operations.sh + +- name: Fix NetworkManager-cloud-setup issue + include_tasks: fix-nm-cloud-setup.yml + when: ansible_distribution_version is version('8.4','=') diff --git a/ansible/playbooks/roles/common/tasks/fix-nm-cloud-setup.yml b/ansible/playbooks/roles/common/tasks/fix-nm-cloud-setup.yml new file mode 100644 index 0000000000..dee5087663 --- /dev/null +++ b/ansible/playbooks/roles/common/tasks/fix-nm-cloud-setup.yml @@ -0,0 +1,41 @@ +--- +# Workaround for bug in NetworkManager-cloud-setup-1.30.0-10.el8_4 (RHEL 8.4) +# +# More info: +# https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/issues/740 +# https://bugzilla.redhat.com/show_bug.cgi?id=2007341 + +- name: Get information on installed packages + package_facts: + manager: rpm + when: ansible_facts.packages is undefined + +# "When a user wants a special network configuration, then it seems reasonable and expected +# that they disable the automatism -- if it doesn't do what they want." +# Source: https://gitlab.freedesktop.org/NetworkManager/NetworkManager/-/merge_requests/974 + +- name: Disable nm-cloud-setup service and remove its routing rule + when: + - ansible_facts.packages['NetworkManager-cloud-setup'] is defined + - ansible_facts.packages['NetworkManager-cloud-setup'][0].version is version('1.30.0', '=') + - ansible_facts.packages['NetworkManager-cloud-setup'][0].release == '10.el8_4' + block: + - name: Disable nm-cloud-setup units + systemd: + name: "{{ item }}" + enabled: false + loop: + - nm-cloud-setup.timer + - nm-cloud-setup.service + + - name: Check if routing rule 30400 exists + command: ip rule list + changed_when: false + register: ip_rule_list + + - name: Remove routing rule 30400 + command: "{{ item }}" + loop: + - ip rule delete prio 30400 + - ip route flush table 30400 + when: "'30400:' in ip_rule_list.stdout" diff --git a/ansible/playbooks/roles/image_registry/defaults/main.yml b/ansible/playbooks/roles/image_registry/defaults/main.yml new file mode 100644 index 0000000000..ecb1a8903c --- /dev/null +++ b/ansible/playbooks/roles/image_registry/defaults/main.yml @@ -0,0 +1,4 @@ +--- +epiphany_registry: + container_name: epiphany-registry + volume_name: epiphany-registry diff --git a/ansible/playbooks/roles/image_registry/tasks/main.yml b/ansible/playbooks/roles/image_registry/tasks/main.yml index 28a11672db..ef16833b7b 100644 --- a/ansible/playbooks/roles/image_registry/tasks/main.yml +++ b/ansible/playbooks/roles/image_registry/tasks/main.yml @@ -17,10 +17,10 @@ - name: Install and configure Docker import_role: name: docker - when: - - check_docker.rc != 0 + when: check_docker.rc != 0 - name: Reconfigure Docker if necessary + when: check_docker.rc == 0 block: - name: Include get-registries.yml from docker role # this sets result include_role: @@ -33,37 +33,65 @@ tasks_from: configure-docker when: - not image_registry_address in result.stdout - when: - - check_docker.rc == 0 - - name: Check if previous version registry is running and stop it if needed - become: true + - name: Migrate registry volume to named one + when: is_upgrade_run block: - - name: Get current running registry image name - command: "{% raw %}docker ps --format '{{ .Image }}' -f 'name=registry' -f 'status=running'{% endraw %}" - register: running_registry_image + - name: Get image name of registry container + command: >- + docker ps --all --format {% raw %}'{{ .Image }}'{% endraw %} -f name={{ epiphany_registry.container_name }} + changed_when: false + register: container_image - - name: Get current running registry container names - command: "{% raw %}docker ps --format '{{ .Names }}' -f 'name=registry' -f 'status=running'{% endraw %}" - register: running_registry_containers + - name: Perform migration when: - - running_registry_image.stdout - - specification.registry_image.name != running_registry_image.stdout - - - set_fact: - container_name: "{{ running_registry_containers.stdout }}" - when: running_registry_containers.changed - - - block: - - name: Stop previous version running registry container - command: docker stop '{{ container_name }}' - - - name: Kill previous version running registry container - command: docker rm '{{ container_name }}' - when: - - container_name is defined - - container_name | length > 0 - when: is_upgrade_run + - container_image.stdout | count > 0 # container exists + - container_image.stdout != specification.registry_image.name + block: + - name: Inspect old version registry container + command: docker inspect {{ epiphany_registry.container_name }} + changed_when: false + register: inspect_registry_container + + - name: Parse information on registry container + set_fact: + container_facts: "{{ inspect_registry_container.stdout | from_json | first }}" + + - name: Check if named volume exists + command: docker volume ls -f name={{ epiphany_registry.volume_name }} --quiet + changed_when: false + register: volume_ls + + - name: Create named volume if not exists + command: docker volume create {{ epiphany_registry.volume_name }} + when: volume_ls.stdout | count == 0 + + - name: Set facts on old version registry volume + set_fact: + old_volume: "{{ container_facts.Mounts | selectattr('Destination', '==' ,'/var/lib/registry') + | first }}" + + - name: Stop old version registry container + command: docker stop {{ epiphany_registry.container_name }} + when: container_facts.State.Status == 'running' + + # There is no 'docker volume rename' command + - name: Move data from old volume + shell: mv {{ old_volume.Source }}/* {{ _new_volume_path }} + args: + removes: "{{ old_volume.Source }}" + vars: + _new_volume_path: >- + {{ old_volume.Source | replace('/' + old_volume.Name + '/', + '/' + epiphany_registry.volume_name + '/') }} + + - name: Remove old version registry container + command: docker rm {{ epiphany_registry.container_name }} + + # Has to be run after container is removed + - name: Remove old version volume + command: docker volume rm {{ old_volume.Name }} + when: old_volume.Name != epiphany_registry.volume_name - name: Load registry image include_tasks: load-image.yml @@ -71,18 +99,28 @@ docker_image: "{{ specification.registry_image }}" - name: Check if registry is running - become: true - command: docker ps -q -f 'ancestor={{ specification.registry_image.name }}' -f 'status=running' - register: current_registry_up_check + command: docker ps -f name={{ epiphany_registry.container_name }} -f status=running --quiet + register: registry_up_check changed_when: false - # todo run registry with SSL - generate/copy certs, mount it to registry container - name: Run registry - become: true - command: >- - docker run -d -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -p 5000:5000 --restart=always - --name epiphany-registry {{ specification.registry_image.name }} - when: current_registry_up_check.stdout | length == 0 + when: registry_up_check.stdout | count == 0 # not running + block: + - name: Check if named volume exists + command: docker volume ls -f name={{ epiphany_registry.volume_name }} --quiet + changed_when: false + register: volume_ls + + - name: Create named volume if not exists + command: docker volume create {{ epiphany_registry.volume_name }} + when: volume_ls.stdout | count == 0 + + # todo run registry with SSL - generate/copy certs, mount it to registry container + - name: Run registry + command: >- + docker run -d -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -p 5000:5000 --restart=always + --name {{ epiphany_registry.container_name }} -v {{ epiphany_registry.volume_name }}:/var/lib/registry + {{ specification.registry_image.name }} - name: Set images to load set_fact: diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index 760c70e1ac..32591a7e38 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -2,11 +2,11 @@ supported_oses: - name: AlmaLinux version: - min: '8.5' + min: '8.4' max: '8.10' - name: RedHat version: - min: '8.5' + min: '8.4' max: '8.10' # based on https://access.redhat.com/support/policy/updates/errata#RHEL8_Life_Cycle - name: Ubuntu version: diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml index f27a82db78..f7d48d5ccf 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/images.yml @@ -175,7 +175,8 @@ images: sha1: be1cf43617eea007629c0eb99149a99b6498f889 'quay.io/ceph/ceph:v16.2.7': - sha1: 13275be4d347e9b305608f52a544e068a73e949f + sha1: fe9b7802c67e19111f83ffe4754ab62df66fd417 + allow_mismatch: true 'quay.io/cephcsi/cephcsi:v3.5.1': sha1: 51dee9ea8ad76fb95ebd16f951e8ffaaaba95eb6 diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml index 36c0271804..9027c32189 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/almalinux-8/packages.yml @@ -7,4 +7,5 @@ prereq-packages: packages: from_repo: [] + multiple_versioned: [] from_url: {} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml index bd4696d4f8..fa1f4cb13c 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/packages.yml @@ -61,6 +61,7 @@ packages: - 'libXcursor' # for grafana - 'libXt' # for grafana - 'logrotate' + - 'lua' # for java-1.8.0-openjdk-headless - 'mcpp' # for grafana - 'net-tools' - 'nfs-utils' @@ -85,6 +86,7 @@ packages: - 'pgaudit15_13-1.5.0' - 'pkgconf' # for bash-completion - 'pkgconf-pkg-config' # for bash-completion + - 'policycoreutils' # for container-selinux - 'python3-cffi' # for python3-cryptography - 'python3-cryptography' - 'python3-firewall' # for firewalld @@ -92,6 +94,7 @@ packages: - 'python3-libselinux' - 'python3-lxml' # for java-1.8.0-openjdk-headless - 'python3-nftables' # for python3-firewall + - 'python3-pip' # for python36 - 'python3-policycoreutils' # for container-selinux - 'python3-psycopg2' - 'python3-pycparser' # for python3-cryptography @@ -120,6 +123,7 @@ packages: - 'xorg-x11-font-utils' # for grafana - 'xorg-x11-server-utils' # for grafana + multiple_versioned: # K8s v1.18.6 (Epiphany >= v0.7.1) - 'kubeadm-1.18.6' - 'kubectl-1.18.6' diff --git a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml index 3e9fca408b..2e6ed4f2ad 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml +++ b/ansible/playbooks/roles/repository/files/download-requirements/requirements/x86_64/redhat/rhel-8/packages.yml @@ -7,4 +7,5 @@ prereq-packages: packages: from_repo: [] + multiple_versioned: [] from_url: {} diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py index fe2e341603..45eaf1032c 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf.py @@ -12,9 +12,9 @@ class Dnf(Command): def __init__(self, retries: int): super().__init__('dnf', retries) - def update(self, enablerepo: str, + def update(self, enablerepo: str = None, package: str = None, - disablerepo: str = '*', + disablerepo: str = None, assume_yes: bool = True): """ Interface for `dnf update` @@ -32,10 +32,21 @@ def update(self, enablerepo: str, if package is not None: update_parameters.append(package) - update_parameters.append(f'--disablerepo={disablerepo}') - update_parameters.append(f'--enablerepo={enablerepo}') + if disablerepo is not None: + update_parameters.append(f'--disablerepo={disablerepo}') + + if enablerepo is not None: + update_parameters.append(f'--enablerepo={enablerepo}') + + proc = self.run(update_parameters) + + if 'error' in proc.stdout: + raise CriticalError( + f'Found an error. dnf update failed for package `{package}`, reason: `{proc.stdout}`') + if proc.stderr: + raise CriticalError( + f'dnf update failed for packages `{package}`, reason: `{proc.stderr}`') - self.run(update_parameters) def install(self, package: str, assume_yes: bool = True): @@ -52,6 +63,13 @@ def install(self, package: str, if not 'does not update' in proc.stdout: # trying to reinstall package with url raise CriticalError(f'dnf install failed for `{package}`, reason `{proc.stdout}`') + if 'error' in proc.stdout: + raise CriticalError( + f'Found an error. dnf install failed for package `{package}`, reason: `{proc.stdout}`') + if proc.stderr: + raise CriticalError( + f'dnf install failed for package `{package}`, reason: `{proc.stderr}`') + def remove(self, package: str, assume_yes: bool = True): """ diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py index c5b9d4c0ec..24f59df960 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_download.py @@ -2,6 +2,7 @@ from typing import List from src.command.command import Command +from src.error import CriticalError class DnfDownload(Command): @@ -21,7 +22,8 @@ def download_packages(self, packages: List[str], args.append(f'--archlist={",".join(archlist)}') args.append(f'--destdir={str(destdir)}') - args.append('--disableplugin=subscription-manager') # to speed up download + # to speed up download + args.append('--disableplugin=subscription-manager') if exclude: args.append(f'--exclude={exclude}') @@ -32,4 +34,10 @@ def download_packages(self, packages: List[str], args.append('-y') args.extend(packages) - self.run(args) + process = self.run(args) + if 'error' in process.stdout: + raise CriticalError( + f'Found an error. dnf download failed for packages `{packages}`, reason: `{process.stdout}`') + if process.stderr: + raise CriticalError( + f'dnf download failed for packages `{packages}`, reason: `{process.stderr}`') diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py index 5aed53c1af..a186e93783 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/command/dnf_repoquery.py @@ -1,3 +1,4 @@ +import re from typing import Callable, List from src.command.command import Command @@ -12,21 +13,25 @@ class DnfRepoquery(Command): def __init__(self, retries: int): super().__init__('dnf', retries) # repoquery would require yum-utils package - def __query(self, packages: List[str], + def __query(self, defined_packages: List[str], queryformat: str, archlist: List[str], requires: bool, resolve: bool, - output_handler: Callable) -> List[str]: + output_handler: Callable, + only_newest: bool = True, + dependencies: bool = False) -> List[str]: """ Run generic query using `dnf repoquery` command. - :param packages: data will be returned for those `packages` + :param defined_packages: data will be returned for those `defined_packages` :param queryformat: specify custom query output format :param archlist: limit results to these architectures - :param requires: get capabilities that the packages depend on + :param requires: get capabilities that the defined_packages depend on :param resolve: resolve capabilities to originating package(s) :param output_handler: different queries produce different outputs, use specific output handler + :param only_newest: if there are more than one candidate packages, download only the newest one + :param dependencies: if it's only to grab dependencies :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`PackageNotfound`: when query did not return any package info @@ -36,10 +41,11 @@ def __query(self, packages: List[str], args.append('repoquery') args.append(f'--archlist={",".join(archlist)}') - args.append('--disableplugin=subscription-manager') # to speed up querying - args.append('--latest-limit=1') + # to speed up querying + args.append('--disableplugin=subscription-manager') + if only_newest: + args.append('--latest-limit=1') args.append(f'--queryformat={queryformat}') - args.append('--quiet') if requires: args.append('--requires') @@ -47,62 +53,91 @@ def __query(self, packages: List[str], if resolve: args.append('--resolve') - args.append('-y') # to import GPG keys - - args.extend(packages) + args.extend(defined_packages) # dnf repoquery doesn't set error code on empty results - output = self.run(args).stdout - output_handler(output) + process = self.run(args) + + output_handler(process.stdout, process.stderr) packages: List[str] = [] - for line in output.split('\n'): + for line in process.stdout.split('\n'): if line: packages.append(line) + if not dependencies: + missing_packages: List[str] = [] + for package in defined_packages: + r = re.compile(f'.*{package}') + match = list(filter(r.match, packages)) + if not match: + missing_packages.append(package) + if missing_packages: + raise PackageNotfound( + f'repoquery failed. Cannot find packages: {missing_packages}') return packages - def query(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]: + def query(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]: """ Generic query to dnf database. :param packages: data will be returned for those `packages` :param queryformat: specify custom query output format :param archlist: limit results to these architectures + :param only_newest: if there are more than one candidate packages, download only the newest one :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`PackageNotfound`: when query did not return any package info :returns: query result """ - def output_handler(output: str): + def output_handler(output_stdout: str, output_stderr: str): """ In addition to errors, handle missing packages """ - if not output: - raise PackageNotfound(f'repoquery failed for packages `{packages}`, reason: some of package(s) not found') - elif 'error' in output: - raise CriticalError(f'repoquery failed for packages `{packages}`, reason: `{output}`') - - return self.__query(packages, queryformat, archlist, False, False, output_handler) - - def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str]) -> List[str]: + if not output_stdout: + raise PackageNotfound( + f'repoquery failed for packages `{packages}`, reason: some of package(s) not found') + if 'error' in output_stdout: + raise CriticalError( + f'Found an error. repoquery failed for packages `{packages}`, reason: `{output_stdout}`') + if "Last metadata expiration check" in output_stderr: + pass # https://dnf.readthedocs.io/en/latest/conf_ref.html#metadata-expire-label + elif "No matches found for the following disable plugin patterns: subscription-manager" in output_stderr: + pass # no subscription-manager on AlmaLinux + else: + raise CriticalError( + f'repoquery failed for packages `{packages}`, reason: `{output_stderr}`') + + return self.__query(packages, queryformat, archlist, False, False, output_handler, only_newest, False) + + def get_dependencies(self, packages: List[str], queryformat: str, archlist: List[str], only_newest: bool = True) -> List[str]: """ Get all dependencies for `packages`. :param packages: data will be returned for those `packages` :param queryformat: specify custom query output format :param archlist: limit results to these architectures + :param only_newest: if there are more than one candidate packages, download only the newest one :raises: :class:`CriticalError`: can be raised on exceeding retries or when error occurred :class:`ValueError`: when `packages` list is empty - :returns: query result + :returns: list of dependencies for `packages` """ # repoquery without KEY argument will query dependencies for all packages if not packages: raise ValueError('packages: list cannot be empty') - def output_handler(output: str): - """ Handle errors """ - if 'error' in output: - raise CriticalError(f'dnf repoquery failed for packages `{packages}`, reason: `{output}`') - - return self.__query(packages, queryformat, archlist, True, True, output_handler) + def output_handler(output_stdout: str, output_stderr: str): + """ In addition to errors, handle missing packages """ + if not output_stdout: + raise PackageNotfound( + f'repoquery failed for packages `{packages}`, reason: some of package(s) not found') + if 'error' in output_stdout: + raise CriticalError( + f'Found an error. repoquery failed for packages `{packages}`, reason: `{output_stdout}`') + if "Last metadata expiration check" in output_stderr: + pass # https://dnf.readthedocs.io/en/latest/conf_ref.html#metadata-expire-label + elif output_stderr: + raise CriticalError( + f'repoquery failed for packages `{packages}`, reason: `{output_stderr}`') + + return self.__query(packages, queryformat, archlist, True, True, output_handler, only_newest, True) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py index 95dda825bf..7dcbe26deb 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/src/mode/red_hat_family_mode.py @@ -2,7 +2,7 @@ import logging import shutil from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Set from src.command.command import Command from src.config.config import Config @@ -16,6 +16,7 @@ class RedHatFamilyMode(BaseMode): def __init__(self, config: Config): super().__init__(config) + self.__all_queried_packages: Set[str] = set() self.__archs: List[str] = [config.os_arch.value, 'noarch'] self.__base_packages: List[str] = ['curl', 'python3-dnf-plugins-core', 'wget'] self.__installed_packages: List[str] = [] @@ -48,6 +49,10 @@ def _create_backup_repositories(self): logging.debug('Done.') def _install_base_packages(self): + + # Bug in RHEL 8.4 https://bugzilla.redhat.com/show_bug.cgi?id=2004853 + self._tools.dnf.update(package='libmodulemd') + # some packages are from EPEL repo # make sure that we reinstall it before proceeding if self._tools.rpm.is_package_installed('epel-release'): @@ -72,7 +77,7 @@ def _install_base_packages(self): def _add_third_party_repositories(self): # Fix for RHUI client certificate expiration [#2318] if self._tools.dnf.is_repo_enabled('rhui-microsoft-azure-rhel'): - self._tools.dnf.update(enablerepo='rhui-microsoft-azure-rhel*') + self._tools.dnf.update(disablerepo='*', enablerepo='rhui-microsoft-azure-rhel*') for repo in self._repositories: repo_filepath = Path('/etc/yum.repos.d') / f'{repo}.repo' @@ -102,6 +107,8 @@ def _add_third_party_repositories(self): '2ndquadrant-dl-default-release-pg13-debug']: self._tools.dnf_config_manager.disable_repo(repo) + self._tools.dnf.makecache(False, True) + def __remove_dnf_cache_for_custom_repos(self): # clean metadata for upgrades (when the same package can be downloaded from changed repo) repocaches: List[str] = list(self.__dnf_cache_path.iterdir()) @@ -136,13 +143,14 @@ def _parse_packages(self) -> Dict[str, Any]: } reqs['packages']['from_repo'] += family_doc['packages']['from_repo'] + reqs['packages']['multiple_versioned'] += family_doc['packages']['multiple_versioned'] # distro level has precedence reqs['packages']['from_url'] = {**family_doc['packages']['from_url'], **distro_doc['packages']['from_url']} return reqs - def __download_prereq_packages(self) -> List[str]: + def __download_prereq_packages(self): # download requirements (fixed versions) prereqs_dir = self._cfg.dest_packages / 'repo-prereqs' prereqs_dir.mkdir(exist_ok=True, parents=True) @@ -159,36 +167,42 @@ def __download_prereq_packages(self) -> List[str]: archlist=self.__archs, exclude='*i686', destdir=prereqs_dir) - return collected_prereqs - def _download_packages(self): - downloaded_prereq_packages: List[str] = self.__download_prereq_packages() - - packages: List[str] = sorted(set(self._requirements['packages']['from_repo'])) + self.__all_queried_packages.update(collected_prereqs) + def __download_redhat_packages(self, packages: List[str], only_newest: bool = True): # packages queried_packages: List[str] = self._tools.repoquery.query(packages, queryformat='%{name}-%{version}-%{release}.%{arch}', - archlist=self.__archs) + archlist=self.__archs, + only_newest=only_newest) - packages_to_download: List[str] = sorted(set(queried_packages) - set(downloaded_prereq_packages)) + packages_to_download: List[str] = sorted(set(queried_packages) - self.__all_queried_packages) logging.info(f'- packages to download: {packages_to_download}') # dependencies dependencies: List[str] = self._tools.repoquery.get_dependencies(packages_to_download, queryformat='%{name}.%{arch}', - archlist=self.__archs) + archlist=self.__archs, + only_newest=only_newest) logging.info(f'- dependencies to download: {dependencies}') packages_to_download = sorted(packages_to_download + dependencies) + self.__all_queried_packages.update(packages_to_download) self._tools.dnf_download.download_packages(packages_to_download, archlist=self.__archs, exclude='*i686', destdir=self._cfg.dest_packages) + def _download_packages(self): + self.__download_prereq_packages() + self.__download_redhat_packages(sorted(set(self._requirements['packages']['from_repo']))) + self.__download_redhat_packages(sorted(set(self._requirements['packages']['multiple_versioned'])), + False) + def _download_file(self, url: str, dest: Path): self._tools.wget.download(url, output_document=dest, additional_params=False) diff --git a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py index 76533c58a7..158509576d 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py +++ b/ansible/playbooks/roles/repository/files/download-requirements/tests/command/test_dnf_repoquery.py @@ -14,8 +14,6 @@ def test_interface_query(mocker): '--disableplugin=subscription-manager', '--latest-limit=1', '--queryformat=some_format', - '--quiet', - '-y', 'tar', 'vim' ] @@ -31,10 +29,8 @@ def test_interface_get_dependencies(mocker): '--disableplugin=subscription-manager', '--latest-limit=1', '--queryformat=some_format', - '--quiet', '--requires', '--resolve', - '-y', 'tar', 'vim' ] diff --git a/cli/src/helpers/naming_helpers.py b/cli/src/helpers/naming_helpers.py index bf5eebaa93..e5b5c36fc7 100644 --- a/cli/src/helpers/naming_helpers.py +++ b/cli/src/helpers/naming_helpers.py @@ -71,8 +71,8 @@ def get_os_name_normalized(vm_doc): if vm_doc.provider == "aws": # Example public/official AMI names: # - ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220419 - # - RHEL-8.5_HVM-20220127-x86_64-3-Hourly2-GP2 - # - AlmaLinux OS 8.5.20211116 x86_64 + # - RHEL-8.4.0_HVM-20210825-x86_64-0-Hourly2-GP2 + # - AlmaLinux OS 8.4.20211015 x86_64 for indicator in expected_indicators: if indicator in vm_doc.specification.os_full_name.lower(): return expected_indicators[indicator] diff --git a/docs/assets/images/lifecycle.png b/docs/assets/images/lifecycle.png deleted file mode 100644 index 54e9c87f0b..0000000000 --- a/docs/assets/images/lifecycle.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:76fdaca8ec741b24864480886f7c929cd17f928f7e91d9abf23e8aadec1273b3 -size 12045 diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index 7aed3d8a40..9e0d740317 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -6,11 +6,11 @@ - [#3080](https://github.com/epiphany-platform/epiphany/issues/3080) - update Filebeat to the latest compatible version with OpenSearch -## [2.0.0] YYYY-MM-DD +## [2.0.0] 2022-05-09 ### Added -- [#1332](https://github.com/epiphany-platform/epiphany/issues/1332) - Support for RHEL 8.5 and AlmaLinux 8.5 +- [#1332](https://github.com/epiphany-platform/epiphany/issues/1332) - Support for RHEL 8.4 and AlmaLinux 8.4 - [#959](https://github.com/epiphany-platform/epiphany/issues/959) - Add usage of use_network_security_groups to disable NSG on AWS - [#2701](https://github.com/epiphany-platform/epiphany/issues/2701) - Epicli prepare - generate files in separate directory - [#2812](https://github.com/epiphany-platform/epiphany/issues/2812) - Extend K8s config validation @@ -95,7 +95,7 @@ ### Breaking changes -- **RHEL 7 and CentOS are not supported**. For RHEL, release 8.5 is required and upgrade from version 7 is not handled by Epiphany. +- **RHEL 7 and CentOS are not supported**. For RHEL, release 8.4 is required and upgrade from version 7 is not handled by Epiphany. Support for CentOS has been replaced by AlmaLinux (migration is not handled by Epiphany). - Support for ARM architecture has been removed (together with CentOS), planned to be added for AlmaLinux in the next release. - Upgrade of Terraform components in issue [#2825](https://github.com/epiphany-platform/epiphany/issues/2825) and [#2853](https://github.com/epiphany-platform/epiphany/issues/2853) will make running re-apply with infrastructure break on existing 1.x clusters. The advice is to deploy a new cluster and migrate data. If needed a manual upgrade path is described [here.](../home/howto/UPGRADE.md#terraform-upgrade-from-epiphany-1.x-to-2.x) diff --git a/docs/home/LIFECYCLE.md b/docs/home/LIFECYCLE.md index 6623f4535b..0f8f5eacac 100644 --- a/docs/home/LIFECYCLE.md +++ b/docs/home/LIFECYCLE.md @@ -33,8 +33,40 @@ The LTS version will be released once a year and will be supported for up to 3 y | [1.1.x STS](../changelogs/CHANGELOG-1.1.md) | 30 Jun 2021 | 1.1.0 | 30 Jun 2021 | 30 Dec 2021 | | [1.2.x STS](../changelogs/CHANGELOG-1.2.md) | 30 Sep 2021 | 1.2.0 | 30 Sep 2021 | 30 Mar 2022 | | [1.3.x STS](../changelogs/CHANGELOG-1.3.md) | 19 Jan 2022 | 1.3.0 | 19 Jan 2022 | 30 Jun 2022 | -| 2.0.x LTS | est. 01 Apr 2022 | - | - | est. 01 Apr 2025 | +| [2.0.x LTS](../changelogs/CHANGELOG-2.0.md) | 09 May 2022 | 2.0.0 | 09 May 2022 | 09 May 2025 | +| :arrow_right: 2.0.1 LTS | est. 30 Jun 2022 | --- | --- | 09 May 2025 | +| :arrow_right: 2.0.2 LTS | est. 30 Sep 2022 | --- | --- | 09 May 2025 | -![lifecycle](../assets/images/lifecycle.png) - -source: [LIFECYCLE_GANTT.md](LIFECYCLE_GANTT.md) +```mermaid +gantt +title Epiphany Platform lifecycle +dateFormat YYYY-MM-DD +section 0.2.x +0.2.x support cycle :a, 2019-02-19, 2020-04-06 +section 0.3.x +0.3.x support cycle :a, 2019-08-02, 2020-07-01 +section 0.4.x +0.4.x support cycle :a, 2019-10-11, 2020-10-22 +section 0.5.x +0.5.x support cycle :a, 2020-01-17, 2021-01-02 +section 0.6.x +0.6.x support cycle :a, 2020-04-06, 2021-04-01 +section 0.7.x +0.7.x support cycle :a, 2020-07-01, 2021-06-30 +section 0.8.x +0.8.x support cycle :a, 2020-10-22, 2021-09-30 +section 0.9.x +0.9.x support cycle :a, 2021-01-19, 2021-12-30 +section 1.0.x +1.0.x support cycle (LTS - 3 years) :crit, 2021-04-01, 2024-04-01 +section 1.1.x +1.1.x - 6 months :a, 2021-06-30, 2021-12-30 +section 1.2.x +1.2.x - 6 months :a, 2021-09-30, 2022-03-30 +section 1.3.x +1.3.x - 6 months :active, 2022-01-19, 2022-06-30 +section 2.0.x +2.0.x support cycle (LTS - 3 years) :crit, 2022-05-09, 2025-05-09 +2.0.1 patch for LTS :crit, 2022-06-30, 2025-05-09 +2.0.2 patch for LTS :crit, 2022-09-30, 2025-05-09 +``` diff --git a/docs/home/LIFECYCLE_GANTT.md b/docs/home/LIFECYCLE_GANTT.md deleted file mode 100644 index ff5d446d66..0000000000 --- a/docs/home/LIFECYCLE_GANTT.md +++ /dev/null @@ -1,40 +0,0 @@ -# Epiphany Platform lifecycle - Gantt chart - -```mermaid -gantt -title Epiphany Platform lifecycle -dateFormat YYYY-MM-DD -section 0.2.x -0.2.x support cycle :done, 2019-02-19, 2020-04-06 -section 0.3.x -0.3.x support cycle :done, 2019-08-02, 2020-07-01 -section 0.4.x -0.4.x support cycle :done, 2019-10-11, 2020-10-22 -section 0.5.x -0.5.x support cycle :done, 2020-01-17, 2021-01-02 -section 0.6.x -0.6.x support cycle :done, 2020-04-06, 2021-04-01 -section 0.7.x -0.7.x support cycle :done, 2020-07-01, 2021-06-30 -section 0.8.x -0.8.x support cycle :done, 2020-10-22, 2021-09-30 -section 0.9.x -0.9.x support cycle :active, 2021-01-19, 2021-12-30 -section 1.0.x -1.0.x support cycle (LTS - 3 years) :crit, 2021-04-01, 2024-04-01 -section 1.1.x -1.1.x - 6 months :active, 2021-06-30, 2021-12-30 -section 1.2.x -1.2.x - 6 months :active, 2021-09-30, 2022-03-30 -section 1.3.x -1.3.x - 6 months :active, 2022-01-19, 2022-06-30 -section 2.0.x -2.0.x support cycle (LTS - 3 years) :crit, 2022-03-30, 2025-03-30 -``` - -This is a source for the image used in [LIFECYCLE.md](LIFECYCLE.md) file. -Currently, Github doesn't support it natively (but feature request was made: [link](https://github.community/t/feature-request-support-mermaid-markdown-graph-diagrams-in-md-files/1922) ). - -Extensions for browsers: -- [Chrome](https://chrome.google.com/webstore/detail/github-%2B-mermaid/goiiopgdnkogdbjmncgedmgpoajilohe) -- [Firefox](https://addons.mozilla.org/en-US/firefox/addon/github-mermaid) diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md index 21f53c7330..5bd2fd1f0d 100644 --- a/docs/home/howto/CLUSTER.md +++ b/docs/home/howto/CLUSTER.md @@ -79,8 +79,8 @@ Epicli has the ability to set up a cluster on infrastructure provided by you. Th At least one of them (with `repository` role) has Internet access in order to download dependencies. If there is no Internet access, you can use [air gap feature (offline mode)](#how-to-create-an-epiphany-cluster-on-existing-air-gapped-infrastructure). 2. The cluster machines/VMs are running one of the following Linux distributions: - - AlmaLinux 8.5+ - - RedHat 7.6+ and < 8 + - AlmaLinux 8.4+ + - RedHat 8.4+ - Ubuntu 20.04 3. The cluster machines/VMs are accessible through SSH with a set of SSH keys you provide and configure on each machine yourself (key-based authentication). 4. The user used for SSH connection (`admin_user`) has passwordless root privileges through `sudo`. @@ -1039,15 +1039,15 @@ sudo chmod +x /home/download-requirements.py # make the requirements script exec After this you should be able to run the ```download-requirements.py``` from the ```home``` folder. -### RedHat 7.x +### RedHat 8.x For RedHat you can use the following command to launch a container: ```shell -docker run -v /shared_folder:/home <--platform linux/amd64 or --platform linux/arm64> --rm -it registry.access.redhat.com/ubi7/ubi:7.9 +docker run -v /shared_folder:/home <--platform linux/amd64 or --platform linux/arm64> --rm -it registry.access.redhat.com/ubi8/ubi:8.4 ``` -As the ```registry.access.redhat.com/ubi7/ubi:7.9``` image is multi-arch you can include ```--platform linux/amd64``` or ```--platform linux/arm64``` to run the container as the specified architecture. The ```/shared_folder``` should be a folder on your local machine containing the requirement scripts. +As the ```registry.access.redhat.com/ubi8/ubi:8.4``` image is multi-arch you can include ```--platform linux/amd64``` or ```--platform linux/arm64``` to run the container as the specified architecture. The ```/shared_folder``` should be a folder on your local machine containing the requirement scripts. For running the ```download-requirements.py``` script you will need a RedHat developer subscription to register the running container and make sure you can access to official Redhat repos for the packages needed. More information on getting this free subscription [here](https://developers.redhat.com/articles/getting-red-hat-developer-subscription-what-rhel-users-need-know). @@ -1059,7 +1059,43 @@ subscription-manager attach --auto # will enable the RedHat official repositorie chmod +x /home/download-requirements.py # make the requirements script executable ``` -After this you should be able to run the ```download-requirements.py``` from the ```home``` folder. +After this you should be able to run the ```download-requirements.py``` from the ```home``` folder: + +```shell +/usr/libexec/platform-python /home/download-requirements.py /home/offline_requirements_rhel_8_x86_64 rhel-8 +``` + +### AlmaLinux 8.x + +For AlmaLinux, you can use the following command to launch a container: + +```shell +docker run -v /shared_folder:/home --rm -it almalinux:8.4 +``` + +The ```almalinux:8.4``` image is amd64 arch only. The ```/shared_folder``` should be a folder on your local machine containing the requirement scripts. + +When you are inside the container run the following command to prepare for the running of the ```download-requirements.py``` script: + +```shell +chmod +x /home/download-requirements.py # make the requirements script executable +``` + +After this you should be able to run the ```download-requirements.py``` from the ```home``` folder: + +```shell +/usr/libexec/platform-python /home/download-requirements.py /home/offline_requirements_almalinux_8_4_x86_64 almalinux-8 +``` + +### Known issues + +In some local environments (eg. using AlmaLinux image) the following issue could appear: + +```sh +Failed to set locale, defaulting to C.UTF-8 +``` + +To fix the issue, verify or set your locales. Example: `export LC_ALL=C.UTF-8` ## How to additional custom Terraform templates diff --git a/docs/home/howto/OS_PATCHING.md b/docs/home/howto/OS_PATCHING.md index 189cd04fe7..e5cbad03a6 100644 --- a/docs/home/howto/OS_PATCHING.md +++ b/docs/home/howto/OS_PATCHING.md @@ -28,7 +28,7 @@ This document will help you decide how you should patch your OS. This is not a s For Epiphany >= v2.0 we recommend the following image (AMI): -- RHEL: `RHEL-8.5_HVM-20220127-x86_64-3-Hourly2-GP2` (kernel 4.18.0-348.12.2.el8_5.x86_64), +- RHEL: `RHEL-8.4.0_HVM-20210825-x86_64-0-Hourly2-GP2` (kernel 4.18.0-305.12.1.el8_4.x86_64), - Ubuntu: `ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220419` (kernel 5.13.0-1022-aws). Note: For different supported OS versions this guide may be useful as well. @@ -56,7 +56,7 @@ For more information, refer to [AWS Systems Manager User Guide](https://docs.aws For Epiphany >= v2.0 we recommend the following image (urn): -- RHEL: `RedHat:rhel-raw:8-raw-gen2:8.5.2022032202` (kernel 4.18.0-348.20.1.el8_5.x86_64), +- RHEL: `RedHat:rhel-raw:8-raw-gen2:8.4.2022031606` (kernel 4.18.0-305.el8.x86_64), - Ubuntu: `Canonical:0001-com-ubuntu-server-focal:20_04-lts-gen2:20.04.202204190` (kernel 5.13.0-1022-azure). Note: For different supported OS versions this guide may be useful as well. diff --git a/docs/home/howto/UPGRADE.md b/docs/home/howto/UPGRADE.md index 1e0843cba3..676c33f9ff 100644 --- a/docs/home/howto/UPGRADE.md +++ b/docs/home/howto/UPGRADE.md @@ -45,8 +45,8 @@ Your airgapped existing cluster should meet the following requirements: 1. The cluster machines/vm`s are connected by a network or virtual network of some sorts and can communicate which each other and have access to the internet: 2. The cluster machines/vm`s are **upgraded** to the following versions: - - AlmaLinux 8.5+ - - RedHat 8.5+ + - AlmaLinux 8.4+ + - RedHat 8.4+ - Ubuntu 20.04 3. The cluster machines/vm`s should be accessible through SSH with a set of SSH keys you provided and configured on each machine yourself. @@ -77,8 +77,8 @@ Your airgapped existing cluster should meet the following requirements: 1. The airgapped cluster machines/vm`s are connected by a network or virtual network of some sorts and can communicate with each other: 2. The airgapped cluster machines/vm`s are **upgraded** to the following versions: - - AlmaLinux 8.5+ - - RedHat 8.5+ + - AlmaLinux 8.4+ + - RedHat 8.4+ - Ubuntu 20.04 3. The airgapped cluster machines/vm`s should be accessible through SSH with a set of SSH keys you provided and configured on each machine yourself. diff --git a/schema/aws/defaults/infrastructure/cloud-os-image-defaults.yml b/schema/aws/defaults/infrastructure/cloud-os-image-defaults.yml index cc70196f9b..caace6d377 100644 --- a/schema/aws/defaults/infrastructure/cloud-os-image-defaults.yml +++ b/schema/aws/defaults/infrastructure/cloud-os-image-defaults.yml @@ -2,7 +2,7 @@ kind: infrastructure/cloud-os-image-defaults title: "Cloud OS Image Defaults" name: default specification: - almalinux-8-arm64: AlmaLinux OS 8.5.20211116 aarch64 - almalinux-8-x86_64: AlmaLinux OS 8.5.20211116 x86_64 - rhel-8-x86_64: RHEL-8.5_HVM-20220127-x86_64-3-Hourly2-GP2 + almalinux-8-arm64: AlmaLinux OS 8.4.20211015 aarch64 + almalinux-8-x86_64: AlmaLinux OS 8.4.20211015 x86_64 + rhel-8-x86_64: RHEL-8.4.0_HVM-20210825-x86_64-0-Hourly2-GP2 ubuntu-20.04-x86_64: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220419 diff --git a/schema/azure/defaults/infrastructure/cloud-os-image-defaults.yml b/schema/azure/defaults/infrastructure/cloud-os-image-defaults.yml index f9fc3359fa..f867b5dc3c 100644 --- a/schema/azure/defaults/infrastructure/cloud-os-image-defaults.yml +++ b/schema/azure/defaults/infrastructure/cloud-os-image-defaults.yml @@ -6,7 +6,7 @@ specification: publisher: almalinux offer: almalinux sku: 8_5-gen2 - version: '8.5.20220311' + version: '8.5.20220311' # offer with PlanId: 8_4 was removed from the marketplace for new purchase plan: name: 8_5-gen2 product: almalinux @@ -15,7 +15,7 @@ specification: publisher: RedHat offer: rhel-raw sku: 8-raw-gen2 - version: '8.5.2022032202' + version: '8.4.2022031606' ubuntu-20.04-x86_64: publisher: Canonical offer: 0001-com-ubuntu-server-focal