Merge branch 'develop' into fix/haproxy-image

.devcontainer/Dockerfile

@@ -4,6 +4,7 @@ ARG USERNAME=vscode
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
+ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
 ARG TERRAFORM_VERSION=1.1.3
@@ -38,7 +39,13 @@ RUN : INSTALL HELM BINARY \
  && curl -fsSLO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
  && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \
  && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
- && terraform version
+ && terraform version \
+ && : INSTALL AWS CLI BINARY \
+ && curl -fsSLO https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip \
+ && unzip awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip \
+ && ./aws/install -i /usr/local/aws-cli -b /usr/local/bin \
+ && rm -rf awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip ./aws \
+ && aws --version
 
 RUN : INSTALL GEM REQUIREMENTS \
  && gem install \

.devcontainer/python.env

@@ -2,4 +2,4 @@
 
 # PYTHONPATH can contain multiple locations separated by os.pathsep: semicolon (;) on Windows and colon (:) on Linux/macOS.
 # Invalid paths are ignored. To verify use "python.analysis.logLevel": "Trace".
-PYTHONPATH=ansible/playbooks/roles/repository/files/download-requirements
+PYTHONPATH=ansible/playbooks/roles/repository/library:ansible/playbooks/roles/repository/files/download-requirements

.vscode/tasks.json

@@ -30,6 +30,16 @@
       "command": "pylint --rcfile .pylintrc ./cli ./tests --output-format text",
       "group": "test",
     },
+    {
+      "label": "Pylint repository modules",
+      "command": "pylint",
+      "args": [
+        "--rcfile", ".pylintrc",
+        "--output-format", "text",
+        "./ansible/playbooks/roles/repository/library/tests",
+      ],
+      "group": "test",
+    },
     {
       "label": "Pylint download-requirements",
       "command": "pylint",

Dockerfile

@@ -4,6 +4,7 @@ ARG USERNAME=epiuser
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
+ARG AWS_CLI_VERSION=2.0.30
 ARG HELM_VERSION=3.3.1
 ARG KUBECTL_VERSION=1.22.4
 ARG TERRAFORM_VERSION=1.1.3
@@ -15,7 +16,7 @@ COPY . /epicli
 RUN : INSTALL APT REQUIREMENTS \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
-        autossh curl gcc jq libcap2-bin libc6-dev libffi-dev make musl-dev openssh-client procps psmisc rsync ruby-full sudo tar unzip vim \
+        autossh curl gcc git jq libcap2-bin libc6-dev libffi-dev make musl-dev openssh-client procps psmisc rsync ruby-full sudo tar unzip vim \
 \
     && : INSTALL HELM BINARY \
     && curl -fsSLO https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \
@@ -32,6 +33,13 @@ RUN : INSTALL APT REQUIREMENTS \
     && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin \
     && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
     && terraform version \
+\
+    && : INSTALL AWS CLI BINARY \
+    && curl -fsSLO https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip \
+    && unzip awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip \
+    && ./aws/install -i /usr/local/aws-cli -b /usr/local/bin \
+    && rm -rf awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip ./aws \
+    && aws --version \
 \
     && : INSTALL GEM REQUIREMENTS \
     && gem install \

ansible/playbooks/roles/repository/files/download-requirements/src/command/debian/apt_cache.py

@@ -66,6 +66,38 @@ def get_package_info(self, package: str, version: str = '') -> Dict[str, str]:
 
         return info
 
+    def __parse_apt_cache_depends(self, stdout: str) -> List[str]:
+        """
+        Parse output from `apt-cache depends`.
+        For deps with alternative, only the first package is choosen.
+        Virtual packages are replaced by the first candidate.
+
+        :param stdout: output from `apt-cache depends` command
+        :returns: required dependencies
+        """
+        alternative_found: bool = False
+        is_alternative: bool = False
+        virt_pkg_found: bool = False
+        deps: List[str] = []
+        for dep in stdout.strip().splitlines():
+
+            dep = dep.replace(' ', '')  # remove white spaces
+
+            if virt_pkg_found and not is_alternative:
+                deps.append(dep)  # pick first from the list
+                virt_pkg_found = False
+
+            if 'Depends:' in dep:  # dependency found
+                is_alternative = alternative_found
+                alternative_found = dep.startswith('|Depends:')
+                virt_pkg_found = '<' in dep and '>' in dep
+
+                if not virt_pkg_found and not is_alternative:
+                    dep = dep.split('Depends:')[-1]  # remove "Depends:
+                    deps.append(dep)
+
+        return deps
+
     def get_package_dependencies(self, package: str, version: str = '') -> List[str]:
         """
         Interface for `apt-cache depends`
@@ -85,38 +117,4 @@ def get_package_dependencies(self, package: str, version: str = '') -> List[str]
                            f'{package}={version}' if version else package]
 
         raw_output = self.run(args).stdout
-
-        virt_pkg: bool = False  # True - virtual package detected, False - otherwise
-        virt_pkgs: List[str] = []  # cached virtual packages options
-        deps: List[str] = []
-        for dep in raw_output.split('\n'):
-            if not dep:  # skip empty lines
-                continue
-
-            dep = dep.replace(' ', '')  # remove white spaces
-
-            if virt_pkg:
-                virt_pkgs.append(dep)  # cache virtual package option
-
-            if '<' in dep and '>' in dep:  # virtual package, more than one dependency to choose
-                virt_pkg = True
-                continue
-
-            if 'Depends:' in dep:  # new dependency found
-                virt_pkg = False
-
-                if virt_pkgs:  # previous choices cached
-                    # avoid conflicts by choosing only non-cached dependency:
-                    if not any(item in deps for item in virt_pkgs):
-                        deps.append(virt_pkgs[0].split('Depends:')[-1])  # pick first from the list
-                    virt_pkgs.clear()
-
-                dep = dep.split('Depends:')[-1]  # remove "Depends:
-
-            if not virt_pkg and dep != package:  # avoid adding package itself
-                # workaround for https://elixirforum.com/t/installing-erlang-elixir-on-ubuntu-20-04-is-failing-esl-erlang-25-0-2-1-ubuntu-focal-amd64-deb-file-has-unexpected-size/48754
-                # TODO: verify after issues 3210 and 3209 are fixed  # pylint: disable=fixme
-                if dep != 'esl-erlang':  # for rabbitmq-server
-                    deps.append(dep)
-
-        return list(set(deps))
+        return self.__parse_apt_cache_depends(raw_output)

ansible/playbooks/roles/repository/files/download-requirements/tests/command/debian/test_apt_cache.py

@@ -1,5 +1,10 @@
+import subprocess
+from unittest.mock import Mock, patch
+
+import pytest
 from src.command.debian.apt_cache import AptCache
 
+from tests.data.apt_cache import APT_CACHE_DEPENDS_RABBITMQ_STDOUT, APT_CACHE_DEPENDS_SOLR_STDOUT
 from tests.mocks.command_run_mock import CommandRunMock
 
 
@@ -16,3 +21,20 @@ def test_interface_get_package_dependencies(mocker):
                              '--no-enhances',
                              '--no-pre-depends',
                              'vim']
+
+
+APT_CACHE_DEPENDS_DATA = [
+    ('tar', 'tar\n', []),
+    ('rabbitmq-server', APT_CACHE_DEPENDS_RABBITMQ_STDOUT, ['adduser', 'erlang-base', 'erlang-crypto', 'python3']),
+    ('solr-common', APT_CACHE_DEPENDS_SOLR_STDOUT, ['curl', 'debconf', 'default-jre-headless', 'libjs-jquery'])]
+
+@pytest.mark.parametrize('PACKAGE_NAME, CMD_STDOUT, EXPECTED_DEPS', APT_CACHE_DEPENDS_DATA)
+def test_get_package_dependencies_return_value(PACKAGE_NAME, CMD_STDOUT, EXPECTED_DEPS):
+    mock_completed_proc = Mock(spec=subprocess.CompletedProcess)
+    mock_completed_proc.returncode = 0
+    mock_completed_proc.stdout = CMD_STDOUT
+
+    with patch('src.command.command.subprocess.run') as mock_run:
+        mock_run.return_value = mock_completed_proc
+        return_value = AptCache(1).get_package_dependencies(package=PACKAGE_NAME)
+        assert return_value == EXPECTED_DEPS

ansible/playbooks/roles/repository/files/download-requirements/tests/data/apt_cache.py

@@ -0,0 +1,32 @@
+APT_CACHE_DEPENDS_RABBITMQ_STDOUT = '''
+rabbitmq-server
+  Depends: adduser
+ |Depends: erlang-base
+  Depends: erlang-base-hipe
+  Depends: erlang-crypto
+  Depends: <python3:any>
+    python3
+    dummy
+'''
+
+APT_CACHE_DEPENDS_SOLR_STDOUT = '''
+solr-common
+  Depends: curl
+  Depends: debconf
+ |Depends: default-jre-headless
+ |Depends: <java5-runtime-headless>
+    default-jre-headless
+    openjdk-11-jre-headless
+    openjdk-13-jre-headless
+    openjdk-16-jre-headless
+    openjdk-17-jre-headless
+    openjdk-8-jre-headless
+  Depends: <java6-runtime-headless>
+    default-jre-headless
+    openjdk-11-jre-headless
+    openjdk-13-jre-headless
+    openjdk-16-jre-headless
+    openjdk-17-jre-headless
+    openjdk-8-jre-headless
+  Depends: libjs-jquery
+'''

ansible/playbooks/roles/repository/files/download-requirements/tests/mocks/command_run_mock.py

@@ -27,22 +27,20 @@ def __enter__(self) -> List[str]:
         """
         :return: list of arguments passed to the subprocess.run() function
         """
-        mock = Mock()
-        mock.returncode = 0
+        mock_completed_proc = Mock(spec=subprocess.CompletedProcess)
+        mock_completed_proc.returncode = 0
 
-        self.__mocker.patch('src.command.command.subprocess.run', side_effect=lambda args, encoding, stdout, stderr: mock)
-
-        spy = self.__mocker.spy(subprocess, 'run')
+        mock_run = self.__mocker.patch('src.command.command.subprocess.run', return_value=mock_completed_proc)
 
         try:
             if self.__args:
                 self.__func(**self.__args)
             else:
                 self.__func()
-        except Exception:
+        except Exception:  # pylint: disable=broad-except
             pass
 
-        return spy.call_args[0][0]
+        return mock_run.call_args[0][0]
 
     def __exit__(self, *args):
         pass