diff --git a/README.md b/README.md index 72ca2cd5e..c5d6bb446 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,9 @@ [![Maven Central](https://img.shields.io/maven-central/v/com.boozallen.aissemble/aissemble-root.svg)](https://search.maven.org/#search%7Cgav%7C1%7Cg%3A%22com.boozallen.aissemble%22%20AND%20a%3A%22aissemble-root%22) ![PyPI](https://img.shields.io/pypi/v/aissemble-foundation-core-python?logo=python&logoColor=gold) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/aissemble-foundation-core-python?logo=python&logoColor=gold) -[![Publish to GitHub Pages](https://github.com/boozallen/aissemble/actions/workflows/publish.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/publish.yml) + +[![Build](https://github.com/boozallen/aissemble/actions/workflows/build.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/build.yml) +[![Publish Docs](https://github.com/boozallen/aissemble/actions/workflows/publish.yml/badge.svg)](https://github.com/boozallen/aissemble/actions/workflows/publish.yml) ## aiSSEMBLE Overview ### Purpose of the aiSSEMBLE diff --git a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/src/aissemble_encrypt/vault_config.py b/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/src/aissemble_encrypt/vault_config.py index 0db552653..c4cde76a0 100644 --- a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/src/aissemble_encrypt/vault_config.py +++ b/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/src/aissemble_encrypt/vault_config.py @@ -64,18 +64,23 @@ def secrets_host_url(self): Returns the secrets host url. This should only be used for integration tests, and not otherwise. """ - return self.properties["secrets.host.url"] + if "SECRETS_HOST_URL" in os.environ: + # host and port of the aissemble-vault container + return os.environ["SECRETS_HOST_URL"] + else: + return self.properties["secrets.host.url"] @staticmethod - def validate_container_start(): + def validate_container_start(port): started = False max_wait = 20 wait = 0 while wait < max_wait: - logger.info("Waiting for Vault to start") + url = f"http://localhost:{port}/v1/sys/health" + logger.info(f"Waiting for Vault to start at {url}") try: - requests.get("http://localhost:8200/v1/sys/health") + requests.get(url) logger.info("Vault started successfully!") started = True break diff --git a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/container/safe_docker_container.py b/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/container/safe_docker_container.py deleted file mode 100644 index eb2c44116..000000000 --- a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/container/safe_docker_container.py +++ /dev/null @@ -1,24 +0,0 @@ -from testcontainers.core.container import DockerContainer - - -class SafeDockerContainer(DockerContainer): - """ - There is currently a bug in the Test Containers library that, due to a race condition, can cause - an attribute error when container objects are being torn down by Python. This just modified the - teardown logic to ensure the attr exists. - - See - """ - - def __init__(self, image: str, **kwargs) -> None: - super().__init__(image, **kwargs) - - def __del__(self) -> None: - """ - Try to remove the container in all circumstances - """ - if hasattr(self, "_container") and self._container is not None: - try: - self.stop() - except: # noqa: E722 - pass diff --git a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/environment.py b/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/environment.py index 75fe38d33..3304ffd6e 100644 --- a/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/environment.py +++ b/extensions/extensions-encryption/aissemble-extensions-encryption-vault-python/tests/features/environment.py @@ -8,8 +8,7 @@ # #L% ### from krausening.logging import LogManager -from container.safe_docker_container import SafeDockerContainer -from testcontainers.core.waiting_utils import wait_for +from testcontainers.core.container import DockerContainer from aissemble_encrypt.vault_config import VaultConfig from importlib import metadata import packaging.version @@ -44,12 +43,14 @@ def select_krausening_extensions(): os.environ["KRAUSENING_EXTENSIONS"] = "tests/resources/krausening/arm64" -def start_container(context, docker_image, feature): +def start_container(context, docker_image, feature) -> int: logger.info(f"Starting container: {docker_image}") - context.test_container.with_bind_ports(8200, 8200) + context.test_container.with_exposed_ports(8200) context.test_container.start() - wait_for(VaultConfig.validate_container_start) - + extport = context.test_container.get_exposed_port(8200) + if not VaultConfig.validate_container_start(extport): + raise Exception("Vault failed to start") + return extport def before_feature(context, feature): if "integration" in feature.tags: @@ -63,8 +64,9 @@ def before_feature(context, feature): version = metadata.version("aissemble-extensions-encryption-vault-python") docker_image += version_to_tag(version) - context.test_container = SafeDockerContainer(docker_image) - start_container(context, docker_image, feature) + context.test_container = DockerContainer(docker_image) + port = start_container(context, docker_image, feature) + os.environ["SECRETS_HOST_URL"] = f"http://127.0.0.1:{port}" root_key_tuple = context.test_container.exec("cat /root_key.txt") secrets_root_key = root_key_tuple.output.decode() diff --git a/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/safe_docker_container.py b/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/safe_docker_container.py deleted file mode 100644 index eb2c44116..000000000 --- a/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/safe_docker_container.py +++ /dev/null @@ -1,24 +0,0 @@ -from testcontainers.core.container import DockerContainer - - -class SafeDockerContainer(DockerContainer): - """ - There is currently a bug in the Test Containers library that, due to a race condition, can cause - an attribute error when container objects are being torn down by Python. This just modified the - teardown logic to ensure the attr exists. - - See - """ - - def __init__(self, image: str, **kwargs) -> None: - super().__init__(image, **kwargs) - - def __del__(self) -> None: - """ - Try to remove the container in all circumstances - """ - if hasattr(self, "_container") and self._container is not None: - try: - self.stop() - except: # noqa: E722 - pass diff --git a/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/testcontainers_kafka_kraft.py b/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/testcontainers_kafka_kraft.py index 4e19c54cd..e9a604e88 100644 --- a/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/testcontainers_kafka_kraft.py +++ b/foundation/foundation-lineage/foundation-data-lineage/aissemble-foundation-data-lineage-python/tests/features/container/testcontainers_kafka_kraft.py @@ -1,11 +1,11 @@ from kafka import KafkaConsumer from kafka.errors import KafkaError, UnrecognizedBrokerVersion, NoBrokersAvailable -from container.safe_docker_container import SafeDockerContainer +from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_container_is_ready -class KafkaKraftContainer(SafeDockerContainer): +class KafkaKraftContainer(DockerContainer): """ Kafka container. diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml index 7f8c2205b..167b00521 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/pom.xml @@ -189,28 +189,6 @@ integration-test - - org.codehaus.mojo - exec-maven-plugin - ${version.exec.maven.plugin} - - - ensure-docker-compose-installed - initialize - - exec - - - sh - - - ${project.basedir}/tests/resources/integration-test-resources/docker-compose-setup.sh - - - - - - maven-resources-plugin diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/environment.py b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/environment.py index b99b971dd..05eaaefe4 100644 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/environment.py +++ b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/environment.py @@ -8,12 +8,15 @@ # #L% ### import os -import platform import json +import time +import packaging.version +from importlib import metadata from pyspark.sql import SparkSession from krausening.logging import LogManager -from testcontainers.compose import DockerCompose +from testcontainers.core.container import DockerContainer from aissemble_test_data_delivery_pyspark_model.generated import environment_base +from aissemble_encrypt.vault_config import VaultConfig """ Behave test environment setup to configure Spark for unit tests. @@ -33,15 +36,6 @@ def before_all(context): print("Created spark session for tests...") - os.environ["S3Test_FS_PROVIDER"] = "s3" - os.environ["S3Test_FS_ACCESS_KEY_ID"] = "000000000000" - os.environ["S3Test_FS_SECRET_ACCESS_KEY"] = ( - "E3FF2839C048B25C084DEBE9B26995E310250568" - ) - os.environ["S3Test_FS_SECURE"] = "False" - os.environ["S3Test_FS_HOST"] = "localhost" - os.environ["S3Test_FS_PORT"] = "4566" - def after_all(context): environment_base.cleanup() @@ -63,47 +57,104 @@ def after_scenario(context, scenario): def before_feature(context, feature): if "integration" in feature.tags: - test_staging_path = "target/generated-sources/docker-compose-files" - logger.info( - f"Starting services defined in Docker Compose file at {test_staging_path}" - ) - - compose = DockerCompose(test_staging_path, "docker-compose.yml") - context.docker_compose_containers = compose - - compose.start() - wait_for_docker_url = "http://localhost:4566" - logger.info( - f"Waiting for Docker Compose services to start - waiting for a response from {wait_for_docker_url}" - ) - compose.wait_for(wait_for_docker_url) + logger.info("Starting Test container services") + context.test_containers = [] + setup_vault(context) + setup_s3_local(context) + + +def setup_vault(context): + docker_image = "ghcr.io/boozallen/aissemble-vault:" + # append current version to docker image + # pyproject.toml has a "version" property, e.g. version = "0.12.0.dev" + # using major, minor, patch and -SNAPSHOT if dev + version = metadata.version("aissemble-extensions-encryption-vault-python") + docker_image += version_to_tag(version) + vault = DockerContainer(docker_image) + port = start_container(vault, 8200, VaultConfig.validate_container_start) + context.test_containers.append(vault) + os.environ["SECRETS_HOST_URL"] = f"http://127.0.0.1:{port}" + + root_key_tuple = vault.exec("cat /root_key.txt") + secrets_root_key = root_key_tuple.output.decode() + os.environ["SECRETS_ROOT_KEY"] = secrets_root_key + + unseal_keys_tuple = vault.exec("cat /unseal_keys.txt") + unseal_keys_txt = unseal_keys_tuple.output.decode() + unseal_keys_json = json.loads(unseal_keys_txt) + secrets_unseal_keys = ",".join(unseal_keys_json) + os.environ["SECRETS_UNSEAL_KEYS"] = secrets_unseal_keys + + transit_client_token_tuple = vault.exec("cat /transit_client_token.txt") + transit_client_token_txt = transit_client_token_tuple.output.decode() + transit_client_token_json = json.loads(transit_client_token_txt) + encrypt_client_token = transit_client_token_json["auth"]["client_token"] + os.environ["ENCRYPT_CLIENT_TOKEN"] = encrypt_client_token + + +def setup_s3_local(context): + localstack = DockerContainer("localstack/localstack:latest") + localstack.with_env("SERVICES", "s3") + port = start_container(localstack, 4566, lambda _: test_aws(localstack)) + context.test_containers.append(localstack) + os.environ["S3Test_FS_PROVIDER"] = "s3" + os.environ["S3Test_FS_ACCESS_KEY_ID"] = "000000000000" + os.environ["S3Test_FS_SECRET_ACCESS_KEY"] = ( + "E3FF2839C048B25C084DEBE9B26995E310250568" + ) + os.environ["S3Test_FS_SECURE"] = "False" + os.environ["S3Test_FS_HOST"] = "localhost" + os.environ["S3Test_FS_PORT"] = f"{port}" - root_key_tuple = context.docker_compose_containers.exec_in_container( - ["cat", "/root_key.txt"], "vault" - ) - secrets_root_key = root_key_tuple[0] - os.environ["SECRETS_ROOT_KEY"] = secrets_root_key - unseal_keys_tuple = context.docker_compose_containers.exec_in_container( - ["cat", "/unseal_keys.txt"], "vault" - ) - unseal_keys_txt = unseal_keys_tuple[0] - unseal_keys_json = json.loads(unseal_keys_txt) - secrets_unseal_keys = ",".join(unseal_keys_json) - os.environ["SECRETS_UNSEAL_KEYS"] = secrets_unseal_keys - - transit_client_token_tuple = ( - context.docker_compose_containers.exec_in_container( - ["cat", "/transit_client_token.txt"], "vault" - ) +def after_feature(context, feature): + if hasattr(context, "test_containers"): + logger.info("Stopping Test container services") + for container in context.test_containers: + logger.info(f"...stopping {container.image}") + container.stop() + + +def start_container(container, port, healthcheck=lambda x: True) -> int: + logger.info(f"Starting container: {container.image}") + container.with_exposed_ports(port) + container.start() + extport = container.get_exposed_port(port) + if not healthcheck(extport): + raise Exception(f"Failed to start {container.image}") + return extport + + +def version_to_tag(version_str: str) -> str: + """Convert a python version into a docker tag for the same version. + + Args: + version_str (str): The version string to convert. + Returns: + str: The docker tag for the version. + """ + version = packaging.version.parse(version_str) + tag = version.base_version + if version.pre: + tag += "-" + "".join([str(x) for x in version.pre]) + if version.is_devrelease: + tag += "-SNAPSHOT" + return tag + + +def test_aws(container): + started = False + tries = 0 + exitcode = -1 + while exitcode != 0 and tries < 20: + logger.info("Waiting for s3 to start...") + tries += 1 + exitcode, _ = container.exec( + "bash -c 'AWS_ACCESS_KEY_ID=fake AWS_SECRET_ACCESS_KEY=fake aws --endpoint-url=http://localhost:4566 s3 ls'" ) - transit_client_token_txt = transit_client_token_tuple[0] - transit_client_token_json = json.loads(transit_client_token_txt) - encrypt_client_token = transit_client_token_json["auth"]["client_token"] - os.environ["ENCRYPT_CLIENT_TOKEN"] = encrypt_client_token + if exitcode == 0: + started = True + else: + time.sleep(1) - -def after_feature(context, feature): - if "integration" in feature.tags: - logger.info("Stopping Docker Compose services") - context.docker_compose_containers.stop() + return started diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose-setup.sh b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose-setup.sh deleted file mode 100644 index a86dab84c..000000000 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose-setup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -### -# #%L -# aiSSEMBLE::Extensions::Encryption::Vault::Python -# %% -# Copyright (C) 2021 Booz Allen -# %% -# This software package is licensed under the Booz Allen Public License. All Rights Reserved. -# #L% -### - -if ! command -v docker-compose &> /dev/null -then - SYS_PLATFORM=$(uname -s | tr '[:upper:]' '[:lower:]') - SYS_ARCH=$(uname -m | tr '[:upper:]' '[:lower:]') - if 'arm64' == "$SYS_ARCH" - then - SYS_ARCH='aarch64' - fi - DOCKER_COMPOSE_RELEASE_URL="https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-${SYS_PLATFORM}-${SYS_ARCH}" - echo "Installing docker-compose v2.12.2 (${SYS_PLATFORM} ${SYS_ARCH})" - echo "Pulling binary from ${DOCKER_COMPOSE_RELEASE_URL}..." - sudo curl -SL "${DOCKER_COMPOSE_RELEASE_URL}" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose -else - echo "Already installed" -fi diff --git a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose.yml b/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose.yml deleted file mode 100644 index ee6f7bc39..000000000 --- a/test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/resources/integration-test-resources/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3' -services: - vault: - # For testing we will pull the image directly from the remote server as opposed to a locally built image. - # This will ensure that the Vault keys are in sync. - image: "ghcr.io/boozallen/aissemble-vault:${version.aissemble}" - cap_add: - - IPC_LOCK - environment: - - VAULT_DEV_ROOT_TOKEN_ID=myroot - - VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200 - ports: - - '8200:8200' - s3-local: - image: localstack/localstack:latest - ports: - - '4566:4566' \ No newline at end of file