Merge pull request #1109 from pulp/update-ci/main

Update CI files for branch main

.ci/ansible/Containerfile.j2

@@ -22,6 +22,7 @@ RUN pip3 install
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt
 
 {% if pulp_env is defined and pulp_env %}
 {% for key, value in pulp_env.items() %}

.ci/assets/ci_constraints.txt

@@ -0,0 +1,7 @@
+# Pulpcore versions without the openapi command do no longer work in the CI
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+
+
+
+tablib!=3.6.0
+# 3.6.0: This release introduced a regression removing the "html" optional dependency.

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-342-g13e7dc5
+2021.08.26-347-gc4a2504

.github/workflows/build.yml

@@ -23,21 +23,26 @@ jobs:
         with:
           fetch-depth: 1
           path: "pulp_deb"
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 1
+          repository: "pulp/pulp-openapi-generator"
+          path: "pulp-openapi-generator"
       - uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install packaging twine wheel
+          pip install packaging twine wheel mkdocs jq
           echo ::endgroup::
       - name: "Build package"
         run: |
           python3 setup.py sdist bdist_wheel --python-tag py3
           twine check dist/*
       - name: "Install built packages"
         run: |
-          pip install dist/pulp_deb-*-py3-none-any.whl
+          pip install dist/pulp_deb-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt
       - name: "Generate api specs"
         run: |
           pulpcore-manager openapi --file "api.json"
@@ -60,3 +65,49 @@ jobs:
           if-no-files-found: "error"
           retention-days: 5
           overwrite: true
+      - name: "Build Python bindings packages"
+        run: |
+          .github/workflows/scripts/build_python_client.sh
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
+      - name: "Upload python client packages"
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "python-client.tar"
+          path: |
+            pulp_deb/deb-python-client.tar
+          if-no-files-found: "error"
+          retention-days: 5
+          overwrite: true
+      - name: "Upload python client docs"
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "python-client-docs.tar"
+          path: |
+            pulp_deb/deb-python-client-docs.tar
+          if-no-files-found: "error"
+          retention-days: 5
+          overwrite: true
+      - name: "Build Ruby bindings packages"
+        run: |
+          .github/workflows/scripts/build_ruby_client.sh
+        shell: "bash"
+        env:
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
+      - name: "Upload Ruby client"
+        uses: "actions/upload-artifact@v4"
+        with:
+          name: "ruby-client.tar"
+          path: |
+            pulp_deb/deb-ruby-client.tar
+          if-no-files-found: "error"
+          retention-days: 5
+          overwrite: true
+...

.github/workflows/scripts/build_python_client.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# This script expects all <app_label>-api.json files to exist in the plugins root directory.
+# It produces a <app_label>-python-client.tar and <app_label>-python-client-docs.tar file in the plugins root directory.
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_deb' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -mveuo pipefail
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
+
+pushd ../pulp-openapi-generator
+rm -rf "pulp_deb-client"
+
+# We need to copy that over to be visible in the container.
+cp "../pulp_deb/deb-api.json" .
+./gen-client.sh "deb-api.json" "deb" python "pulp_deb"
+
+pushd pulp_deb-client
+python setup.py sdist bdist_wheel --python-tag py3
+
+twine check "dist/pulp_deb_client-"*"-py3-none-any.whl"
+twine check "dist/pulp_deb-client-"*".tar.gz"
+
+tar cvf "../../pulp_deb/deb-python-client.tar" ./dist
+
+find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \;
+find ./docs/* -exec sed -i 's/README//g' {} \;
+cp README.md docs/index.md
+sed -i 's/docs\///g' docs/index.md
+find ./docs/* -exec sed -i 's/\.md//g' {} \;
+
+cat >> mkdocs.yml << DOCSYAML
+---
+site_name: PulpDeb Client
+site_description: Deb bindings
+site_author: Pulp Team
+site_url: https://docs.pulpproject.org/pulp_deb_client/
+repo_name: pulp/pulp_deb
+repo_url: https://github.com/pulp/pulp_deb
+theme: readthedocs
+DOCSYAML
+
+# Building the bindings docs
+mkdocs build
+
+# Pack the built site.
+tar cvf ../../pulp_deb/deb-python-client-docs.tar ./site
+popd
+popd

.github/workflows/scripts/build_ruby_client.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# This script expects all <app_label>-api.json files to exist in the plugins root directory.
+# It produces a <app_label>-ruby-client.tar file in the plugins root directory.
+
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_deb' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+set -mveuo pipefail
+
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
+
+pushd ../pulp-openapi-generator
+rm -rf "pulp_deb-client"
+
+# We need to copy that over to be visible in the container.
+#cp "../pulp_deb/deb-api.json" .
+#./gen-client.sh "deb-api.json" "deb" ruby "pulp_deb"
+
+# -------------
+# The generator still needs to have it called api.json at this time...
+cp "../pulp_deb/api.json" .
+./gen-client.sh "api.json" "deb" ruby "pulp_deb"
+# -------------
+
+pushd pulp_deb-client
+gem build pulp_deb_client
+tar cvf "../../pulp_deb/deb-ruby-client.tar" "./pulp_deb_client-"*".gem"
+popd
+popd

.github/workflows/scripts/script.sh

@@ -54,49 +54,74 @@ password password
 # Some commands like ansible-galaxy specifically require 600
 cmd_prefix bash -c "chmod 600 ~pulp/.netrc"
 
-# Generate and install binding
+# Generate bindings
+###################
+
+echo "::group::Generate bindings"
+
+touch bindings_requirements.txt
 pushd ../pulp-openapi-generator
-# Use app_label to generate api.json and package to produce the proper package name.
+  # Use app_label to generate api.json and package to produce the proper package name.
 
-if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ]
-then
   # Workaround: Domains are not supported by the published bindings.
-  # Generate new bindings for all packages.
-  for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS")
-  do
-    echo $item
-    COMPONENT="$(jq -r '.component' <<<"$item")"
-    VERSION="$(jq -r '.version' <<<"$item")"
-    MODULE="$(jq -r '.module' <<<"$item")"
-    PACKAGE="${MODULE%%.*}"
-    cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > api.json
-    ./gen-client.sh api.json "${COMPONENT}" python "${PACKAGE}"
-    cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
-    sudo rm -rf "./${PACKAGE}-client"
-  done
-else
   # Sadly: Different pulpcore-versions aren't either...
-  for item in $(jq -r '.versions[]| select(.component!="deb")| tojson' <<<"$REPORTED_STATUS")
+  # So we exclude the prebuilt ones only for domains disabled.
+  if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ]
+  then
+    BUILT_CLIENTS=""
+  else
+    BUILT_CLIENTS=" deb "
+  fi
+
+  for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}")
   do
-    echo $item
-    COMPONENT="$(jq -r '.component' <<<"$item")"
-    VERSION="$(jq -r '.version' <<<"$item")"
-    MODULE="$(jq -r '.module' <<<"$item")"
+    COMPONENT="$(jq -r '.component' <<<"${ITEM}")"
+    VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")"
+    # On older status endpoints, the module was not provided, but the package should be accurate
+    # there, because we did not merge plugins into pulpcore back then.
+    MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")"
     PACKAGE="${MODULE%%.*}"
-    cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > api.json
-    ./gen-client.sh api.json "${COMPONENT}" python "${PACKAGE}"
-    cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
-    sudo rm -rf "./${PACKAGE}-client"
+    cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json"
+    if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]]
+    then
+      rm -rf "./${PACKAGE}-client"
+      ./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}"
+      pushd "${PACKAGE}-client"
+        python setup.py sdist bdist_wheel --python-tag py3
+      popd
+    else
+      if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ]
+      then
+        ls -lR "${PACKAGE}-client/"
+        echo "Error: Client bindings for ${COMPONENT} not found."
+        echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing."
+        exit 1
+      fi
+    fi
+    echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_deb/bindings_requirements.txt"
   done
-fi
 popd
 
-# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format.
-echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt
+echo "::endgroup::"
+
+echo "::group::Debug bindings diffs"
+
+# Bindings diff for deb
+jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "deb-api.json" > "build-api.json"
+jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/deb-api.json" > "test-api.json"
+jsondiff --indent 2 build-api.json test-api.json || true
+echo "::endgroup::"
+
+# Install test requirements
+###########################
+
+# Add a safeguard to make sure the proper versions of the clients are installed.
+echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt
 cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt
 cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt
 cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt
-cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt
+cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt
+cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt
 
 CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
 cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'"
@@ -127,7 +152,7 @@ else
   else
     cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_deb.tests.functional -m parallel -n 8"
     cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_deb.tests.functional -m 'not parallel'"
-  fi 
+  fi
 fi
 export PULP_FIXTURES_URL="http://pulp-fixtures:8080"
 pushd ../pulp-cli-deb