From 379301264e122609ede86d55eb57a86d5fae4ce8 Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Tue, 21 Sep 2021 19:07:43 -0300 Subject: [PATCH] Enable azure tests closes #501 --- CHANGES/501.feature | 1 + README.md | 7 ++-- plugin-template | 1 + .../github/.ci/ansible/Containerfile.j2.copy | 3 ++ .../github/.ci/ansible/settings.py.j2.copy | 11 ++++++ templates/github/.github/workflows/ci.yml.j2 | 3 ++ .../github/.github/workflows/nightly.yml.j2 | 3 ++ .../github/.github/workflows/release.yml.j2 | 19 ++++++----- .../workflows/scripts/before_script.sh.j2 | 2 +- .../.github/workflows/scripts/install.sh.j2 | 34 ++++++++++++++++++- 10 files changed, 72 insertions(+), 12 deletions(-) create mode 100644 CHANGES/501.feature diff --git a/CHANGES/501.feature b/CHANGES/501.feature new file mode 100644 index 00000000..bc2b800d --- /dev/null +++ b/CHANGES/501.feature @@ -0,0 +1 @@ +Enable azure tests diff --git a/README.md b/README.md index db4e8a1f..3b09769a 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The following settings are stored in `template_config.yml`. bindings (optional): boolean whether or not generate bindings org (optional): github organization revision (optional): the git commit hash to install. - "Required PR" in a commit message will override this. + "Required PR" in a commit message will override this. branch: the git branch of the plugin. applies to non-tagged GHA jobs, such as PRs & cron jobs. pip_version_specifier: A pip version specifier for when the gets installed from PyPI. Applies to TAGGED (release) jobs. @@ -83,7 +83,7 @@ The following settings are stored in `template_config.yml`. check_stray_pulpcore_imports Check that plugins are importing from pulpcore.plugin and not pulpcore directly. - + core_import_allowed A list of string patterns to be allowed to import from pulpcore explicitly. coverage Include collection of coverage and reporting to coveralls.io @@ -231,6 +231,9 @@ The following settings are stored in `template_config.yml`. see if it is compatible with pulpcore's master branch. This helps ensure that pulpcore is following the deprecation policy for the plugin API. + test_azure Include azure job for running tests using [azurite](https://github.com/Azure/Azurite) + to emulate Azure. + test_s3 Include s3 job for running tests using [minio](https://github.com/minio/minio) to emulate S3. diff --git a/plugin-template b/plugin-template index 4b7a54b7..68abed32 100755 --- a/plugin-template +++ b/plugin-template @@ -66,6 +66,7 @@ DEFAULT_SETTINGS = { "test_cli": False, "test_performance": False, "test_released_plugin_with_next_pulpcore_release": False, + "test_azure": False, "test_s3": False, "update_redmine": False, "ci_trigger": "{pull_request: {branches: ['*']}}", diff --git a/templates/github/.ci/ansible/Containerfile.j2.copy b/templates/github/.ci/ansible/Containerfile.j2.copy index eaff66a8..4483eab8 100644 --- a/templates/github/.ci/ansible/Containerfile.j2.copy +++ b/templates/github/.ci/ansible/Containerfile.j2.copy @@ -14,6 +14,9 @@ RUN pip3 install \ {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} +{%- if azure_test | default(false) -%} +{{ " " }}django-storages[azure] +{%- endif -%} {%- for item in plugins -%} {%- if item.name == "pulp-certguard" -%} {{ " " }}python-dateutil rhsm diff --git a/templates/github/.ci/ansible/settings.py.j2.copy b/templates/github/.ci/ansible/settings.py.j2.copy index e0e040a7..068ab831 100644 --- a/templates/github/.ci/ansible/settings.py.j2.copy +++ b/templates/github/.ci/ansible/settings.py.j2.copy @@ -24,3 +24,14 @@ AWS_S3_ENDPOINT_URL = "http://minio:9000" DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" AWS_DEFAULT_ACL = "@none None" {% endif %} + +{% if azure_test | default(false) %} +AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +AZURE_ACCOUNT_NAME = "devstoreaccount1" +AZURE_CONTAINER = "pulp-test" +AZURE_LOCATION = "pulp3" +AZURE_OVERWRITE_FILES = True +AZURE_URL_EXPIRATION_SECS = 120 +AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;' +DEFAULT_FILE_STORAGE = "storages.backends.azure_storage.AzureStorage" +{% endif %} diff --git a/templates/github/.github/workflows/ci.yml.j2 b/templates/github/.github/workflows/ci.yml.j2 index 80a6e15e..0df8f2fd 100644 --- a/templates/github/.github/workflows/ci.yml.j2 +++ b/templates/github/.github/workflows/ci.yml.j2 @@ -97,6 +97,9 @@ jobs: {%- if docs_test %} - TEST: docs {%- endif %} + {%- if test_azure %} + - TEST: azure + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/nightly.yml.j2 b/templates/github/.github/workflows/nightly.yml.j2 index e1242be1..4cfc28eb 100644 --- a/templates/github/.github/workflows/nightly.yml.j2 +++ b/templates/github/.github/workflows/nightly.yml.j2 @@ -30,6 +30,9 @@ jobs: {%- if docs_test %} - TEST: docs {%- endif %} + {%- if test_azure %} + - TEST: azure + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/release.yml.j2 b/templates/github/.github/workflows/release.yml.j2 index 3444da86..27058f5c 100644 --- a/templates/github/.github/workflows/release.yml.j2 +++ b/templates/github/.github/workflows/release.yml.j2 @@ -1,14 +1,14 @@ {% include 'header.j2' %} -{% from 'macros.j2' import - set_env_vars, - checkout, - setup_python, +{% from 'macros.j2' import + set_env_vars, + checkout, + setup_python, setup_ruby, - install_httpie, - after_failure, - run_script, + install_httpie, + after_failure, + run_script, set_secrets, - install_python_deps, + install_python_deps, configure_git, with context %} --- @@ -70,6 +70,9 @@ jobs: {%- if docs_test %} - TEST: docs {%- endif %} + {%- if test_azure %} + - TEST: azure + {%- endif %} {%- if test_s3 %} - TEST: s3 {%- endif %} diff --git a/templates/github/.github/workflows/scripts/before_script.sh.j2 b/templates/github/.github/workflows/scripts/before_script.sh.j2 index bdff0169..2cdd2a4a 100755 --- a/templates/github/.github/workflows/scripts/before_script.sh.j2 +++ b/templates/github/.github/workflows/scripts/before_script.sh.j2 @@ -24,7 +24,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml echo "PULP CONFIG:" tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == 'azure' || "$TEST" == "plugin-from-pypi" ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core fi diff --git a/templates/github/.github/workflows/scripts/install.sh.j2 b/templates/github/.github/workflows/scripts/install.sh.j2 index 72c02f8f..b97b09b8 100755 --- a/templates/github/.github/workflows/scripts/install.sh.j2 +++ b/templates/github/.github/workflows/scripts/install.sh.j2 @@ -123,7 +123,7 @@ VARSYAML {%- if docker_fixtures %} -if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then +if [[ "$TEST" == "pulp" || "$TEST" == "performance" || "$TEST" == "upgrade" || "$TEST" == "azure" || "$TEST" == "s3" || "$TEST" == "plugin-from-pypi" ]]; then sed -i -e '/^services:/a \ - name: pulp-fixtures\ image: docker.io/pulp/pulp-fixtures:latest\ @@ -149,6 +149,25 @@ minio_secret_key: "'$MINIO_SECRET_KEY'"' vars/main.yaml fi {%- endif %} +{%- if test_azure %} + +if [ "$TEST" = "azure" ]; then + mkdir -p azurite + cd azurite + openssl req -newkey rsa:2048 -x509 -nodes -keyout azkey.pem -new -out azcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:pulp-azurite" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN" + sudo cp azcert.pem /usr/local/share/ca-certificates/azcert.crt + sudo dpkg-reconfigure ca-certificates + cd .. + sed -i -e '/^services:/a \ + - name: pulp-azurite\ + image: mcr.microsoft.com/azure-storage/azurite\ + volumes:\ + - ./azurite:/etc/pulp\ + command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml + sed -i -e '$a azure_test: true' vars/main.yaml +fi +{%- endif %} + ansible-playbook build_container.yaml ansible-playbook start_container.yaml @@ -160,6 +179,9 @@ sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certi # Hack: adding pulp CA to certifi.where() CERTIFI=$(python -c 'import certifi; print(certifi.where())') cat /usr/local/share/ca-certificates/pulp_webserver.crt | sudo tee -a $CERTIFI +if [ "$TEST" = "azure" ]; then + cat /usr/local/share/ca-certificates/azcert.crt | sudo tee -a $CERTIFI +fi # Hack: adding pulp CA to default CA file CERT=$(python -c 'import ssl; print(ssl.get_default_verify_paths().openssl_cafile)') @@ -170,6 +192,16 @@ sudo update-ca-certificates echo ::endgroup:: {%- endif %} +if [ "$TEST" = "azure" ]; then + {%- if pulp_scheme == "https" %} + cat /usr/local/share/ca-certificates/azcert.crt >> /opt/az/lib/python3.6/site-packages/certifi/cacert.pem + cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /usr/local/lib/python3.8/site-packages/certifi/cacert.pem" + cat /usr/local/share/ca-certificates/azcert.crt | docker exec -i pulp bash -c "cat >> /etc/pki/tls/cert.pem" + {%- endif %} + AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://pulp-azurite:10000/devstoreaccount1;' + az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING +fi + echo ::group::PIP_LIST cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" echo ::endgroup::