Update plugin template

This accommodates for a change in pulpcore that will effect the CI.

pulp/pulpcore#1102

[noissue]

.ci/ansible/Containerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ ci_base | default("pulp/pulp-ci:latest") }}
+FROM {{ ci_base | default("pulp/pulp-ci-centos:latest") }}
 
 # Add source directories to container
 {% for item in plugins %}
@@ -10,7 +10,7 @@ ADD {{ item.source }} {{ item.source }}
 # Install python packages
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install --use-feature=2020-resolver \
+RUN pip3 install \
 {%- if s3_test | default(false) -%}
 {{ " " }}django-storages[boto3] git+https://github.com/fabricio-aguiar/botocore.git@fix-100-continue
 {%- endif -%}
@@ -23,7 +23,7 @@ RUN pip3 install --use-feature=2020-resolver \
 
 RUN mkdir -p /etc/nginx/pulp/
 {% for item in plugins %}
-RUN ln /usr/local/lib/python3.7/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
+RUN ln /usr/local/lib/python3.6/site-packages/{{ item.name }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
 {% endfor %}
 
 ENTRYPOINT ["/init"]

.ci/ansible/start_container.yaml

@@ -81,13 +81,22 @@
           command: "docker logs pulp"
           failed_when: true
 
-    - name: "Check version of component being tested"
-      assert:
-        that:
-          - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver)
-        fail_msg: |
-          Component {{ component_name }} was expected to be installed in version {{ component_version }}.
-          Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}.
+    - block:
+        - name: "Check version of component being tested"
+          assert:
+            that:
+              - (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] | canonical_semver == (component_version | canonical_semver)
+            fail_msg: |
+              Component {{ component_name }} was expected to be installed in version {{ component_version }}.
+              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[component_name] }}.
+      rescue:
+        - name: "Check version of component being tested (legacy)"
+          assert:
+            that:
+              - (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] | canonical_semver == (component_version | canonical_semver)
+            fail_msg: |
+              Component {{ legacy_component_name }} was expected to be installed in version {{ component_version }}.
+              Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[legacy_component_name] }}.
 
     - name: "Set pulp password in .netrc"
       copy:

.ci/scripts/docs-builder.py

@@ -13,6 +13,9 @@
 import re
 from shutil import rmtree
 import tempfile
+import requests
+import json
+from packaging import version
 
 WORKING_DIR = os.environ["WORKSPACE"]
 
@@ -90,8 +93,17 @@ def main():
 
     ga_build = False
 
+    publish_at_root = False
+
     if (not re.search("[a-zA-Z]", branch) or "post" in branch) and len(branch.split(".")) > 2:
         ga_build = True
+        # Only publish docs at the root if this is the latest version
+        r = requests.get("https://pypi.org/pypi/pulp-npm/json")
+        latest_version = version.parse(json.loads(r.text)["info"]["version"])
+        docs_version = version.parse(branch)
+        if latest_version == docs_version:
+            publish_at_root = True
+        # Post releases should use the x.y.z part of the version string to form a path
         if "post" in branch:
             branch = ".".join(branch.split(".")[:-1])
 
@@ -122,22 +134,23 @@ def main():
     elif ga_build:
         # This is a GA build.
         # publish to the root of docs.pulpproject.org
-        version_components = branch.split(".")
-        x_y_version = "{}.{}".format(version_components[0], version_components[1])
-        remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT)
-        rsync_command = [
-            "rsync",
-            "-avzh",
-            "--delete",
-            "--exclude",
-            "en",
-            "--omit-dir-times",
-            local_path_arg,
-            remote_path_arg,
-        ]
-        exit_code = subprocess.call(rsync_command, cwd=docs_directory)
-        if exit_code != 0:
-            raise RuntimeError("An error occurred while pushing docs.")
+        if publish_at_root:
+            version_components = branch.split(".")
+            x_y_version = "{}.{}".format(version_components[0], version_components[1])
+            remote_path_arg = "%s@%s:%s" % (USERNAME, HOSTNAME, SITE_ROOT)
+            rsync_command = [
+                "rsync",
+                "-avzh",
+                "--delete",
+                "--exclude",
+                "en",
+                "--omit-dir-times",
+                local_path_arg,
+                remote_path_arg,
+            ]
+            exit_code = subprocess.call(rsync_command, cwd=docs_directory)
+            if exit_code != 0:
+                raise RuntimeError("An error occurred while pushing docs.")
         # publish to docs.pulpproject.org/en/3.y/
         make_directory_with_rsync(["en", x_y_version])
         remote_path_arg = "%s@%s:%sen/%s/" % (

.ci/scripts/release.py

@@ -7,6 +7,7 @@
 
 import argparse
 import json
+import re
 import os
 import textwrap
 from collections import defaultdict
@@ -74,7 +75,7 @@ def validate_redmine_data(redmine_query_url, redmine_issues):
 with open(f"{plugin_path}/setup.py") as fp:
     for line in fp.readlines():
         if "version=" in line:
-            version = line.split('"')[1]
+            version = re.split("\"|'", line)[1]
     if not version:
         raise RuntimeError("Could not detect existing version ... aborting.")
 release_version = version.replace(".dev", "")
@@ -186,7 +187,7 @@ def validate_redmine_data(redmine_query_url, redmine_issues):
 with open(f"{plugin_path}/setup.py") as fp:
     for line in fp.readlines():
         if "version=" in line:
-            new_dev_version = line.split('"')[1]
+            new_dev_version = re.split("\"|'", line)[1]
     if not new_dev_version:
         raise RuntimeError("Could not detect new dev version ... aborting.")
 

.ci/scripts/schema.py

@@ -0,0 +1,23 @@
+"""
+Customizing OpenAPI validation.
+
+OpenAPI requires paths to start with slashes:
+https://spec.openapis.org/oas/v3.0.3#patterned-fields
+
+But some pulp paths start with curly brackets e.g. {artifact_href}
+This script modifies drf-spectacular schema validation to accept slashes and curly brackets.
+"""
+import json
+from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH
+
+with open(JSON_SCHEMA_SPEC_PATH) as fh:
+    openapi3_schema_spec = json.load(fh)
+
+properties = openapi3_schema_spec["definitions"]["Paths"]["patternProperties"]
+# Making OpenAPI validation to accept paths starting with / and {
+if "^\\/|{" not in properties:
+    properties["^\\/|{"] = properties["^\\/"]
+    del properties["^\\/"]
+
+with open(JSON_SCHEMA_SPEC_PATH, "w") as fh:
+    json.dump(openapi3_schema_spec, fh)

.github/workflows/scripts/before_install.sh

@@ -34,7 +34,8 @@ else
 fi
 mkdir .ci/ansible/vars || true
 echo "---" > .ci/ansible/vars/main.yaml
-echo "component_name: pulp_npm" >> .ci/ansible/vars/main.yaml
+echo "legacy_component_name: pulp_npm" >> .ci/ansible/vars/main.yaml
+echo "component_name: npm" >> .ci/ansible/vars/main.yaml
 echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
 
 export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
@@ -57,11 +58,13 @@ then
   export PULPCORE_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
   export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
   export PULP_OPENAPI_GENERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
+  export PULP_CLI_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-cli\/pull\/(\d+)' | awk -F'/' '{print $7}')
   echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
 else
   export PULPCORE_PR_NUMBER=
   export PULP_SMASH_PR_NUMBER=
   export PULP_OPENAPI_GENERATOR_PR_NUMBER=
+  export PULP_CLI_PR_NUMBER=
   export CI_BASE_IMAGE=
 fi
 
@@ -80,6 +83,8 @@ sed -i -e 's/localhost:24817/pulp/g' generate.sh
 sed -i -e 's/:24817/pulp/g' generate.sh
 cd ..
 
+
+
 git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master
 
 cd pulpcore

.github/workflows/scripts/script.sh

@@ -31,6 +31,11 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then
   make PULP_URL="http://pulp" html
   cd ..
 
+  echo "Validating OpenAPI schema..."
+  cat $PWD/.ci/scripts/schema.py | cmd_stdin_prefix bash -c "cat > /tmp/schema.py"
+  cmd_prefix bash -c "python3 /tmp/schema.py"
+  cmd_prefix bash -c "pulpcore-manager spectacular --file pulp_schema.yml --validate"
+
   if [ -f $POST_DOCS_TEST ]; then
     source $POST_DOCS_TEST
   fi
@@ -85,11 +90,20 @@ echo "Checking for uncommitted migrations..."
 cmd_prefix bash -c "django-admin makemigrations --check --dry-run"
 
 # Run unit tests.
-cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.7/site-packages/pulp_npm/tests/unit/"
+cmd_prefix bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python3.6/site-packages/pulp_npm/tests/unit/"
 
 # Run functional tests
 export PYTHONPATH=$REPO_ROOT:$REPO_ROOT/../pulpcore${PYTHONPATH:+:${PYTHONPATH}}
 
+if [[ "$TEST" == "performance" ]]; then
+  if [[ -z ${PERFORMANCE_TEST+x} ]]; then
+    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_npm.tests.performance
+  else
+    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_npm.tests.performance.test_$PERFORMANCE_TEST
+  fi
+  exit
+fi
+
 if [ -f $FUNC_TEST_SCRIPT ]; then
   source $FUNC_TEST_SCRIPT
 else

template_config.yml

@@ -34,6 +34,7 @@ pypi_username: pulp
 redmine_project: npm-plugin
 stable_branch: null
 test_bindings: false
+test_cli: false
 test_performance: false
 test_released_plugin_with_next_pulpcore_release: false
 test_s3: true