diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index 87f58daf..d6b3a8c0 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -1,4 +1,4 @@ -FROM {{ ci_base | default("pulp/pulp-ci:latest") }} +FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }} # Add source directories to container {% for item in plugins %} @@ -10,7 +10,7 @@ ADD {{ item.source }} {{ item.source }} # Install python packages # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install --use-feature=2020-resolver \ +RUN pip3 install \ {%- if s3_test | default(false) -%} {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue {%- endif -%} @@ -23,7 +23,7 @@ RUN pip3 install --use-feature=2020-resolver \ RUN mkdir -p /etc/nginx/pulp/ {% for item in plugins %} -RUN ln /usr/local/lib/python3.7/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true +RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true {% endfor %} ENTRYPOINT ["/init"] diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index 43d4d470..bdf35b43 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -81,13 +81,22 @@ command: "docker logs pulp" failed_when: true - - name: "Check version of component being tested" - assert: - that: - - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) - fail_msg: | - Component {{ component_name }} was expected to be installed in version {{ component_version }}. - Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. + - block: + - name: "Check version of component being tested" + assert: + that: + - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver) + fail_msg: | + Component {{ component_name }} was expected to be installed in version {{ component_version }}. + Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}. + rescue: + - name: "Check version of component being tested (legacy)" + assert: + that: + - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver) + fail_msg: | + Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}. + Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}. - name: "Set pulp password in .netrc" copy: diff --git a/.ci/scripts/docs-builder.py b/.ci/scripts/docs-builder.py index a9a6d7d5..85566c91 100644 --- a/.ci/scripts/docs-builder.py +++ b/.ci/scripts/docs-builder.py @@ -13,6 +13,9 @@ import re from shutil import rmtree import tempfile +import requests +import json +from packaging import version WORKING_DIR = os.environ["WORKSPACE"] @@ -90,8 +93,17 @@ def main(): ga_build = False + publish_at_root = False + if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2: ga_build = True + # Only publish docs at the root if this is the latest version + r = requests.get("https://pypi.org/pypi/pulp-certguard/json") + latest_version = version.parse(json.loads(r.text)["info"]["version"]) + docs_version = version.parse(branch) + if latest_version == docs_version: + publish_at_root = True + # Post releases should use the x.y.z part of the version string to form a path if "post" in branch: branch = ".".join(branch.split(".")[:-1]) @@ -122,22 +134,23 @@ def main(): elif ga_build: # This is a GA build. # publish to the root of docs.pulpproject.org - version_components = branch.split(".") - x_y_version = "{}.{}".format(version_components[0], version_components[1]) - remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) - rsync_command = [ - "rsync", - "-avzh", - "--delete", - "--exclude", - "en", - "--omit-dir-times", - local_path_arg, - remote_path_arg, - ] - exit_code = subprocess.call(rsync_command, cwd=docs_directory) - if exit_code != 0: - raise RuntimeError("An error occurred while pushing docs.") + if publish_at_root: + version_components = branch.split(".") + x_y_version = "{}.{}".format(version_components[0], version_components[1]) + remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT) + rsync_command = [ + "rsync", + "-avzh", + "--delete", + "--exclude", + "en", + "--omit-dir-times", + local_path_arg, + remote_path_arg, + ] + exit_code = subprocess.call(rsync_command, cwd=docs_directory) + if exit_code != 0: + raise RuntimeError("An error occurred while pushing docs.") # publish to docs.pulpproject.org/en/3.y/ make_directory_with_rsync(["en", x_y_version]) remote_path_arg = "%s@%s:%sen/%s/" % ( diff --git a/.ci/scripts/release.py b/.ci/scripts/release.py index 3ce13b33..56995c2d 100644 --- a/.ci/scripts/release.py +++ b/.ci/scripts/release.py @@ -7,6 +7,7 @@ import argparse import json +import re import os import textwrap from collections import defaultdict @@ -74,7 +75,7 @@ def validate_redmine_data(redmine_query_url, redmine_issues): with open(f"{plugin_path}/setup.py") as fp: for line in fp.readlines(): if "version=" in line: - version = line.split('"')[1] + version = re.split("\"|'", line)[1] if not version: raise RuntimeError("Could not detect existing version ... aborting.") release_version = version.replace(".dev", "") @@ -186,7 +187,7 @@ def validate_redmine_data(redmine_query_url, redmine_issues): with open(f"{plugin_path}/setup.py") as fp: for line in fp.readlines(): if "version=" in line: - new_dev_version = line.split('"')[1] + new_dev_version = re.split("\"|'", line)[1] if not new_dev_version: raise RuntimeError("Could not detect new dev version ... aborting.") diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py new file mode 100644 index 00000000..9f56caa6 --- /dev/null +++ b/.ci/scripts/schema.py @@ -0,0 +1,23 @@ +""" +Customizing OpenAPI validation. + +OpenAPI requires paths to start with slashes: +https://spec.openapis.org/oas/v3.0.3#patterned-fields + +But some pulp paths start with curly brackets e.g. {artifact_href} +This script modifies drf-spectacular schema validation to accept slashes and curly brackets. +""" +import json +from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH + +with open(JSON_SCHEMA_SPEC_PATH) as fh: + openapi3_schema_spec = json.load(fh) + +properties = openapi3_schema_spec["definitions"]["Paths"]["patternProperties"] +# Making OpenAPI validation to accept paths starting with / and { +if "^\\/|{" not in properties: + properties["^\\/|{"] = properties["^\\/"] + del properties["^\\/"] + +with open(JSON_SCHEMA_SPEC_PATH, "w") as fh: + json.dump(openapi3_schema_spec, fh) diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 45308343..b17f7b16 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -34,7 +34,8 @@ else fi mkdir .ci/ansible/vars || true echo "---" > .ci/ansible/vars/main.yaml -echo "component_name: pulp_certguard" >> .ci/ansible/vars/main.yaml +echo "legacy_component_name: pulp_certguard" >> .ci/ansible/vars/main.yaml +echo "component_name: certguard" >> .ci/ansible/vars/main.yaml echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh @@ -57,12 +58,14 @@ then export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}') export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}') export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}') + export PULP_CLI_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-cli\/pull\/(\d+)' | awk -F'/' '{print $7}') export PULP_FILE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp_file\/pull\/(\d+)' | awk -F'/' '{print $7}') echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml else export PULPCORE_PR_NUMBER= export PULP_SMASH_PR_NUMBER= export PULP_OPENAPI_GENERATOR_PR_NUMBER= + export PULP_CLI_PR_NUMBER= export PULP_FILE_PR_NUMBER= export CI_BASE_IMAGE= fi @@ -82,6 +85,8 @@ sed -i -e 's/localhost:24817/pulp/g' generate.sh sed -i -e 's/:24817/pulp/g' generate.sh cd .. + + git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master cd pulpcore diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 2f730478..58599d96 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -31,6 +31,11 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then make PULP_URL="http://pulp" html cd .. + echo "Validating OpenAPI schema..." + cat $PWD/.ci/scripts/schema.py | cmd_stdin_prefix bash -c "cat > /tmp/schema.py" + cmd_prefix bash -c "python3 /tmp/schema.py" + cmd_prefix bash -c "pulpcore-manager spectacular --file pulp_schema.yml --validate" + if [ -f $POST_DOCS_TEST ]; then source $POST_DOCS_TEST fi @@ -87,11 +92,20 @@ echo "Checking for uncommitted migrations..." cmd_prefix bash -c "django-admin makemigrations --check --dry-run" # Run unit tests. -cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.7/site-packages/pulp_certguard/tests/unit/" +cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulp_certguard/tests/unit/" # Run functional tests export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}} +if [[ "$TEST" == "performance" ]]; then + if [[ -z ${PERFORMANCE_TEST+x} ]]; then + pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_certguard.tests.performance + else + pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_certguard.tests.performance.test_$PERFORMANCE_TEST + fi + exit +fi + if [ -f $FUNC_TEST_SCRIPT ]; then source $FUNC_TEST_SCRIPT else diff --git a/template_config.yml b/template_config.yml index b20e6a1c..dd55f6c4 100644 --- a/template_config.yml +++ b/template_config.yml @@ -36,6 +36,7 @@ pypi_username: pulp redmine_project: certguard stable_branch: null test_bindings: false +test_cli: false test_performance: false test_released_plugin_with_next_pulpcore_release: false test_s3: false