diff --git a/.devcontainer/python.env b/.devcontainer/python.env index ed94d63408..729b2ec774 100644 --- a/.devcontainer/python.env +++ b/.devcontainer/python.env @@ -2,4 +2,4 @@ # PYTHONPATH can contain multiple locations separated by os.pathsep: semicolon (;) on Windows and colon (:) on Linux/macOS. # Invalid paths are ignored. To verify use "python.analysis.logLevel": "Trace". -PYTHONPATH=ansible/playbooks/roles/repository/files/download-requirements +PYTHONPATH=ansible/playbooks/roles/repository/library:ansible/playbooks/roles/repository/files/download-requirements diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 78c92161f4..13d0ad2938 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -30,6 +30,16 @@ "command": "pylint --rcfile .pylintrc ./cli ./tests --output-format text", "group": "test", }, + { + "label": "Pylint repository modules", + "command": "pylint", + "args": [ + "--rcfile", ".pylintrc", + "--output-format", "text", + "./ansible/playbooks/roles/repository/library/tests", + ], + "group": "test", + }, { "label": "Pylint download-requirements", "command": "pylint", diff --git a/ansible/playbooks/roles/repository/library/__init__.py b/ansible/playbooks/roles/repository/library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/playbooks/roles/repository/library/filter_credentials.py b/ansible/playbooks/roles/repository/library/filter_credentials.py new file mode 100644 index 0000000000..dced8574db --- /dev/null +++ b/ansible/playbooks/roles/repository/library/filter_credentials.py @@ -0,0 +1,154 @@ +#!/usr/bin/python + +from __future__ import (absolute_import, division, print_function) + +from hashlib import sha256 +from pathlib import Path +from typing import Callable +import yaml + +__metaclass__ = type + + +DOCUMENTATION = r""" +--- +module: filter_credentials + +short_description: Module for filtering sensitive data stored in manifest.yml file + +options: + src: + description: Path to the manifest file that will be filtered + required: true + type: str + dest: + description: Path to the newly created, filtered manifest + required: false + type: str +""" + +EXAMPLES = r""" +# Pass in a manifest file without modifying the original file +- name: Filter out manifest file and set result to stdout + filter_credentials: + src: /some/where/manifest.yml + +# Pass in a manifest file and save it to `dest` location +- name: Filter out manifest file and save it as a new file + filter_credentials: + src: /some/where/manifest.yml + dest: /some/other/place/manifest.yml +""" + + +from ansible.module_utils.basic import AnsibleModule + + +def _get_hash(filepath: Path) -> str: + # calculate sha256 for `filepath` + with filepath.open(mode='rb') as file_handler: + hashgen = sha256() + hashgen.update(file_handler.read()) + return hashgen.hexdigest() + + +def _filter_common(docs: list[dict]): + # remove admin user info from epiphany-cluster doc: + try: + del next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs))['specification']['admin_user'] + except KeyError: + pass # ok, key already doesn't exist + + +def _filter_aws(docs: list[dict]): + _filter_common(docs) + + # filter epiphany-cluster doc + epiphany_cluster = next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs)) + + try: + del epiphany_cluster['specification']['cloud']['credentials'] + except KeyError: + pass # ok, key already doesn't exist + + +def _filter_azure(docs: list[dict]): + _filter_common(docs) + + # filter epiphany-cluster doc + epiphany_cluster = next(filter(lambda doc: doc['kind'] == 'epiphany-cluster', docs)) + try: + del epiphany_cluster['specification']['cloud']['subscription_name'] + except KeyError: + pass # ok, key already doesn't exist + + +def _get_filtered_manifest(manifest_path: Path) -> str: + """ + Load the manifest file and remove any sensitive data. + + :param manifest_path: manifest file which will be loaded + :returns: filtered manifset + """ + docs = yaml.safe_load_all(manifest_path.open()) + filtered_docs = [doc for doc in docs if doc['kind'] in ['epiphany-cluster', + 'configuration/feature-mappings', + 'configuration/features', + 'configuration/image-registry']] + + FILTER_DATA: dict[str, Callable] = { + 'any': _filter_common, + 'azure': _filter_azure, + 'aws': _filter_aws + } + + FILTER_DATA[filtered_docs[0]['provider']](filtered_docs) + + return yaml.dump_all(filtered_docs) + +def run_module(): + # define available arguments/parameters a user can pass to the module + module_args = dict( + src=dict(type=str, required=True), + dest=dict(type=str, required=False, default=None), + ) + + # seed the result dict in the object + result = dict( + changed=False, + manifest='' + ) + + # create ansible module + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + input_manifest = Path(module.params['src']) + output_manifest = Path(module.params['dest']) if module.params['dest'] else None + + manifest = _get_filtered_manifest(input_manifest) + + if not module.params['dest']: # to stdout + result['manifest'] = manifest + else: # write to a new location + orig_hash_value = _get_hash(input_manifest) # hash value prior to change + + with output_manifest.open(mode='w', encoding='utf-8') as output_manifest_file: + output_manifest_file.write(manifest) + + new_hash_value = _get_hash(output_manifest) # hash value post change + + if orig_hash_value != new_hash_value: + result['changed'] = True + + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/ansible/playbooks/roles/repository/library/tests/__init__.py b/ansible/playbooks/roles/repository/library/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py b/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py new file mode 100644 index 0000000000..84c9c5864b --- /dev/null +++ b/ansible/playbooks/roles/repository/library/tests/data/filter_credentials_data.py @@ -0,0 +1,267 @@ +CLUSTER_DOC_ANY = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'any', + 'name': 'default', + 'specification': { + 'name': 'test', + 'admin_user': { + 'name': 'operations', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'components': { + 'repository': { + 'count': 1, + 'machines': ['default-repository']}, + 'kubernetes_master': { + 'count': 1, + 'machines': ['default-k8s-master1']}, + 'kubernetes_node': { + 'count': 2, + 'machines': ['default-k8s-node1', 'default-k8s-node2']}, + 'logging': { + 'count': 1, + 'machines': ['default-logging']}, + 'monitoring': { + 'count': 1, + 'machines': ['default-monitoring']}, + 'kafka': { + 'count': 2, + 'machines': ['default-kafka1', 'default-kafka2']}, + 'postgresql': { + 'count': 1, + 'machines': ['default-postgresql']}, + 'load_balancer': { + 'count': 1, + 'machines': ['default-loadbalancer']}, + 'rabbitmq': { + 'count': 1, + 'machines': ['default-rabbitmq']}, + 'opensearch': { + 'count': 1, + 'machines': ['default-opensearch']} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_ANY = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'any', + 'name': 'default', + 'specification': { + 'name': 'test', + 'components': { + 'repository': { + 'count': 1, + 'machines': ['default-repository']}, + 'kubernetes_master': { + 'count': 1, + 'machines': ['default-k8s-master1']}, + 'kubernetes_node': { + 'count': 2, + 'machines': ['default-k8s-node1', 'default-k8s-node2']}, + 'logging': { + 'count': 1, + 'machines': ['default-logging']}, + 'monitoring': { + 'count': 1, + 'machines': ['default-monitoring']}, + 'kafka': { + 'count': 2, + 'machines': ['default-kafka1', 'default-kafka2']}, + 'postgresql': { + 'count': 1, + 'machines': ['default-postgresql']}, + 'load_balancer': { + 'count': 1, + 'machines': ['default-loadbalancer']}, + 'rabbitmq': { + 'count': 1, + 'machines': ['default-rabbitmq']}, + 'opensearch': { + 'count': 1, + 'machines': ['default-opensearch']} + } + }, + 'version': '2.0.1dev' +} + + +CLUSTER_DOC_AZURE = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'azure', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'admin_user': { + 'name': 'operations', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'cloud': { + 'subscription_name': 'YOUR-SUB-NAME', + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default'}, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_AZURE = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'azure', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default'}, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + +CLUSTER_DOC_AWS = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'aws', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'admin_user': { + 'name': 'ubuntu', + 'key_path': '/shared/.ssh/epiphany-operations/id_rsa'}, + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'credentials': { + 'access_key_id': 'XXXX-XXXX-XXXX', + 'secret_access_key': 'XXXXXXXXXXXXXXXX'}, + 'default_os_image': 'default' + }, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +EXPECTED_CLUSTER_DOC_AWS = { + 'kind': 'epiphany-cluster', + 'title': 'Epiphany cluster Config', + 'provider': 'aws', + 'name': 'default', + 'specification': { + 'name': 'test', + 'prefix': 'prefix', + 'cloud': { + 'k8s_as_cloud_service': False, + 'use_public_ips': False, + 'default_os_image': 'default' + }, + 'components': { + 'repository': {'count': 1}, + 'kubernetes_master': {'count': 1}, + 'kubernetes_node': {'count': 2}, + 'logging': {'count': 1}, + 'monitoring': {'count': 1}, + 'kafka': {'count': 2}, + 'postgresql': {'count': 1}, + 'load_balancer': {'count': 1}, + 'rabbitmq': {'count': 1}, + 'opensearch': {'count': 1} + } + }, + 'version': '2.0.1dev' +} + + +COMMON_DOCS = [ + { + 'kind': 'configuration/feature-mappings', + 'title': 'Feature mapping to components', + 'name': 'default' + }, + { + 'kind': 'configuration/image-registry', + 'title': 'Epiphany image registry', + 'name': 'default' + } +] + + +NOT_NEEDED_DOCS = [ + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-loadbalancer', + 'specification': { + 'hostname': 'loadbalancer', + 'ip': '192.168.100.110' + }, + 'version': '2.0.1dev' + }, + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-rabbitmq', + 'specification': { + 'hostname': 'rabbitmq', + 'ip': '192.168.100.111' + }, + 'version': '2.0.1dev' + }, + { + 'kind': 'infrastructure/machine', + 'provider': 'any', + 'name': 'default-opensearch', + 'specification': { + 'hostname': 'opensearch', + 'ip': '192.168.100.112' + }, + 'version': '2.0.1dev' + } +] + + +MANIFEST_WITH_ADDITIONAL_DOCS = [ CLUSTER_DOC_ANY ] + COMMON_DOCS + NOT_NEEDED_DOCS diff --git a/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py b/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py new file mode 100644 index 0000000000..2d4c350a4a --- /dev/null +++ b/ansible/playbooks/roles/repository/library/tests/test_filter_credentials.py @@ -0,0 +1,41 @@ +from copy import deepcopy # make sure that objects used in tests don't get damaged in between test cases +from pathlib import Path + +import pytest + +from library.filter_credentials import _get_filtered_manifest + +from library.tests.data.filter_credentials_data import ( + CLUSTER_DOC_ANY, + CLUSTER_DOC_AWS, + CLUSTER_DOC_AZURE, + EXPECTED_CLUSTER_DOC_ANY, + EXPECTED_CLUSTER_DOC_AWS, + EXPECTED_CLUSTER_DOC_AZURE, + MANIFEST_WITH_ADDITIONAL_DOCS +) + + +@pytest.mark.parametrize('CLUSTER_DOC, EXPECTED_OUTPUT_DOC', + [(CLUSTER_DOC_ANY, EXPECTED_CLUSTER_DOC_ANY), + (CLUSTER_DOC_AZURE, EXPECTED_CLUSTER_DOC_AZURE), + (CLUSTER_DOC_AWS, EXPECTED_CLUSTER_DOC_AWS)]) +def test_epiphany_cluster_doc_filtering(CLUSTER_DOC, EXPECTED_OUTPUT_DOC, mocker): + # Ignore yaml parsing, work on python objects: + mocker.patch('library.filter_credentials.yaml.safe_load_all', return_value=[deepcopy(CLUSTER_DOC)]) + mocker.patch('library.filter_credentials.yaml.dump_all', side_effect=lambda docs: docs) + mocker.patch('library.filter_credentials.Path.open') + + assert _get_filtered_manifest(Path('')) == [EXPECTED_OUTPUT_DOC] + + +def test_not_needed_docs_filtering(mocker): + # Ignore yaml parsing, work on python objects: + mocker.patch('library.filter_credentials.yaml.safe_load_all', return_value=deepcopy(MANIFEST_WITH_ADDITIONAL_DOCS)) + mocker.patch('library.filter_credentials.yaml.dump_all', side_effect=lambda docs: docs) + mocker.patch('library.filter_credentials.Path.open') + + EXPECTED_DOCS = ['epiphany-cluster', 'configuration/feature-mappings', 'configuration/image-registry'] + FILTERED_DOCS = [doc['kind'] for doc in _get_filtered_manifest(Path(''))] + + assert FILTERED_DOCS == EXPECTED_DOCS diff --git a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml index 16b72da464..43650fd901 100644 --- a/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml +++ b/ansible/playbooks/roles/repository/tasks/copy-download-requirements.yml @@ -55,11 +55,19 @@ dest: "{{ download_requirements_dir }}/{{ item }}" loop: "{{ _files }}" - - name: Copy the manifest file - synchronize: - src: "{{ input_manifest_path }}" - dest: "{{ download_requirements_dir }}/manifest.yml" + - name: Manifest handling when: not full_download and input_manifest_path + block: + - name: Filter sensitive data from the manifest + filter_credentials: + src: "{{ input_manifest_path }}" + dest: /tmp/filtered_manifest.yml + delegate_to: localhost + + - name: Copy the manifest file + synchronize: + src: /tmp/filtered_manifest.yml + dest: "{{ download_requirements_dir }}/manifest.yml" - name: Copy RedHat family specific download requirements file synchronize: diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md index d5b476a92c..bb6a2c882c 100644 --- a/docs/changelogs/CHANGELOG-2.0.md +++ b/docs/changelogs/CHANGELOG-2.0.md @@ -15,6 +15,7 @@ - [#3140](https://github.com/epiphany-platform/epiphany/issues/3140) - Allow to disable OpenSearch audit logs - [#3218](https://github.com/epiphany-platform/epiphany/issues/3218) - Add support for original output coloring - [#3079](https://github.com/epiphany-platform/epiphany/issues/3079) - OpenSearch improvement - add dedicated user for Filebeat +- [#3207](https://github.com/epiphany-platform/epiphany/issues/3207) - Add filtering mechanism for the sensitive data ### Fixed diff --git a/pytest.ini b/pytest.ini index 042744a4e9..0d7b30de6e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,4 +5,5 @@ filterwarnings = ignore:The distutils package is deprecated:DeprecationWarning:packaging.tags testpaths = tests/unit/ + ansible/playbooks/roles/repository/library/tests/ ansible/playbooks/roles/repository/files/download-requirements/tests/