diff --git a/ansible/playbooks/roles/repository/library/filter_credentials.py b/ansible/playbooks/roles/repository/library/filter_credentials.py index 370bf9ecdd..603489c207 100644 --- a/ansible/playbooks/roles/repository/library/filter_credentials.py +++ b/ansible/playbooks/roles/repository/library/filter_credentials.py @@ -3,7 +3,8 @@ from __future__ import (absolute_import, division, print_function) from pathlib import Path -from typing import List +from typing import Callable, Dict, List +import yaml __metaclass__ = type @@ -26,7 +27,7 @@ in_place: description: When set to True will modify existing manifest passed as `src` required: false - type: false + type: bool """ EXAMPLES = r""" @@ -49,35 +50,51 @@ """ -import yaml from ansible.module_utils.basic import AnsibleModule +def _filter_common(docs: List[Dict]): + # remove admin user info from epiphany-cluster doc: + del next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs))['specification']['admin_user'] + + +def _filter_aws(docs: List[Dict]): + _filter_common(docs) + + # filter epiphany-cluster doc + epiphany_cluster = next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs)) + del epiphany_cluster['specification']['cloud']['credentials'] + + +def _filter_azure(docs: List[Dict]): + _filter_common(docs) + + # filter epiphany-cluster doc + epiphany_cluster = next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs)) + del epiphany_cluster['specification']['cloud']['subscription_name'] + + def _get_filtered_manifest(manifest_path: Path) -> str: """ Load the manifest file and remove any sensitive data. :param manifest_path: manifest file which will be loaded - :returns: filtered manifset doc + :returns: filtered manifset """ + docs = yaml.safe_load_all(manifest_path.open()) + filtered_docs = [doc for doc in docs if doc['kind'] in ['epiphany-cluster', + 'configuration/feature-mappings', + 'configuration/image-registry']] - # remove passwords: - lines: List[str] = [] - with manifest_path.open(mode='r', encoding='utf-8') as mhandler: - for line in mhandler.readlines(): - if 'password' not in line.lower(): - lines.append(line) - - docs = yaml.safe_load_all(''.join(lines)) - epiphany_cluster = next(docs) + FILTER_DATA: Dict[str, Callable] = { + 'any': _filter_common, + 'azure': _filter_azure, + 'aws': _filter_aws + } - # remove provider credentials: - try: - del epiphany_cluster['specification']['cloud'] - except KeyError: - pass # cloud key is optional + FILTER_DATA[filtered_docs[0]['provider']](filtered_docs) - return yaml.dump_all([epiphany_cluster] + list(docs)) + return yaml.dump_all(filtered_docs) def run_module(): # define available arguments/parameters a user can pass to the module diff --git a/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py b/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py new file mode 100644 index 0000000000..84c9c5864b --- /dev/null +++ b/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py @@ -0,0 +1,267 @@ +CLUSTER_DOC_ANY = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'any', + 'name': 'default', + 'specification': { + 'name': 'test', + 'admin_user': { + 'name': 'operations', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'components': { + 'repository': { + 'count': 1, + 'machines': ['default-repository']}, + 'kubernetes_master': { + 'count': 1, + 'machines': ['default-k8s-master1']}, + 'kubernetes_node': { + 'count': 2, + 'machines': ['default-k8s-node1', 'default-k8s-node2']}, + 'logging': { + 'count': 1, + 'machines': ['default-logging']}, + 'monitoring': { + 'count': 1, + 'machines': ['default-monitoring']}, + 'kafka': { + 'count': 2, + 'machines': ['default-kafka1', 'default-kafka2']}, + 'postgresql': { + 'count': 1, + 'machines': ['default-postgresql']}, + 'load_balancer': { + 'count': 1, + 'machines': ['default-loadbalancer']}, + 'rabbitmq': { + 'count': 1, + 'machines': ['default-rabbitmq']}, + 'opensearch': { + 'count': 1, + 'machines': ['default-opensearch']} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_ANY = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'any', + 'name': 'default', + 'specification': { + 'name': 'test', + 'components': { + 'repository': { + 'count': 1, + 'machines': ['default-repository']}, + 'kubernetes_master': { + 'count': 1, + 'machines': ['default-k8s-master1']}, + 'kubernetes_node': { + 'count': 2, + 'machines': ['default-k8s-node1', 'default-k8s-node2']}, + 'logging': { + 'count': 1, + 'machines': ['default-logging']}, + 'monitoring': { + 'count': 1, + 'machines': ['default-monitoring']}, + 'kafka': { + 'count': 2, + 'machines': ['default-kafka1', 'default-kafka2']}, + 'postgresql': { + 'count': 1, + 'machines': ['default-postgresql']}, + 'load_balancer': { + 'count': 1, + 'machines': ['default-loadbalancer']}, + 'rabbitmq': { + 'count': 1, + 'machines': ['default-rabbitmq']}, + 'opensearch': { + 'count': 1, + 'machines': ['default-opensearch']} + } + }, + 'version': '2.0.1dev' +} + + +CLUSTER_DOC_AZURE = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'azure', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'admin_user': { + 'name': 'operations', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'cloud': { + 'subscription_name': 'YOUR-SUB-NAME', + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default'}, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_AZURE = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'azure', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default'}, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + +CLUSTER_DOC_AWS = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'aws', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'admin_user': { + 'name': 'ubuntu', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'credentials': { + 'access_key_id': 'XXXX-XXXX-XXXX', + 'secret_access_key': 'XXXXXXXXXXXXXXXX'}, + 'default_os_image': 'default' + }, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_AWS = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'aws', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default' + }, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +COMMON_DOCS = [ + { + 'kind': 'configuration/feature-mappings', + 'title': 'Feature mapping to components', + 'name': 'default' + }, + { + 'kind': 'configuration/image-registry', + 'title': 'Epiphany image registry', + 'name': 'default' + } +] + + +NOT_NEEDED_DOCS = [ + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-loadbalancer', + 'specification': { + 'hostname': 'loadbalancer', + 'ip': '192.168.100.110' + }, + 'version': '2.0.1dev' + }, + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-rabbitmq', + 'specification': { + 'hostname': 'rabbitmq', + 'ip': '192.168.100.111' + }, + 'version': '2.0.1dev' + }, + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-opensearch', + 'specification': { + 'hostname': 'opensearch', + 'ip': '192.168.100.112' + }, + 'version': '2.0.1dev' + } +] + + +MANIFEST_WITH_ADDITIONAL_DOCS = [ CLUSTER_DOC_ANY ] + COMMON_DOCS + NOT_NEEDED_DOCS diff --git a/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py b/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py new file mode 100644 index 0000000000..5f709f8e4b --- /dev/null +++ b/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py @@ -0,0 +1,41 @@ +from copy import deepcopy # make sure that objects used in tests don't get damaged in between test cases +from pathlib import Path + +import pytest + +from repository.library.filter_credentials import _get_filtered_manifest + +from repository.library.tests.data.filter_credentials_data import ( + CLUSTER_DOC_ANY, + CLUSTER_DOC_AWS, + CLUSTER_DOC_AZURE, + EXPECTED_CLUSTER_DOC_ANY, + EXPECTED_CLUSTER_DOC_AWS, + EXPECTED_CLUSTER_DOC_AZURE, + MANIFEST_WITH_ADDITIONAL_DOCS +) + + +@pytest.mark.parametrize('CLUSTER_DOC, EXPECTED_OUTPUT_DOC', + [(CLUSTER_DOC_ANY, EXPECTED_CLUSTER_DOC_ANY), + (CLUSTER_DOC_AZURE, EXPECTED_CLUSTER_DOC_AZURE), + (CLUSTER_DOC_AWS, EXPECTED_CLUSTER_DOC_AWS)]) +def test_epiphany_cluster_doc_filtering(CLUSTER_DOC, EXPECTED_OUTPUT_DOC, mocker): + # Ignore yaml parsing, work on python objects: + mocker.patch('repository.library.filter_credentials.yaml.safe_load_all', return_value=[deepcopy(CLUSTER_DOC)]) + mocker.patch('repository.library.filter_credentials.yaml.dump_all', side_effect=lambda docs: docs) + mocker.patch('repository.library.filter_credentials.Path.open') + + assert _get_filtered_manifest(Path('')) == [EXPECTED_OUTPUT_DOC] + + +def test_not_needed_docs_filtering(mocker): + # Ignore yaml parsing, work on python objects: + mocker.patch('repository.library.filter_credentials.yaml.safe_load_all', return_value=deepcopy(MANIFEST_WITH_ADDITIONAL_DOCS)) + mocker.patch('repository.library.filter_credentials.yaml.dump_all', side_effect=lambda docs: docs) + mocker.patch('repository.library.filter_credentials.Path.open') + + EXPECTED_DOCS = ['epiphany-cluster', 'configuration/feature-mappings', 'configuration/image-registry'] + FILTERED_DOCS = [doc['kind'] for doc in _get_filtered_manifest(Path(''))] + + assert FILTERED_DOCS == EXPECTED_DOCS diff --git a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml index 186620e824..43650fd901 100644 --- a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml @@ -55,18 +55,19 @@ dest: "{{ download_requirements_dir }}/{{ item }}" loop: "{{ _files }}" - - name: Filter sensitive data from the manifest - filter_credentials: - src: "{{ input_manifest_path }}" - dest: /tmp/filtered_manifest.yml - delegate_to: localhost + - name: Manifest handling when: not full_download and input_manifest_path + block: + - name: Filter sensitive data from the manifest + filter_credentials: + src: "{{ input_manifest_path }}" + dest: /tmp/filtered_manifest.yml + delegate_to: localhost - - name: Copy the manifest file - synchronize: - src: /tmp/filtered_manifest.yml - dest: "{{ download_requirements_dir }}/manifest.yml" - when: not full_download and input_manifest_path + - name: Copy the manifest file + synchronize: + src: /tmp/filtered_manifest.yml + dest: "{{ download_requirements_dir }}/manifest.yml" - name: Copy RedHat family specific download requirements file synchronize: