Update CI files

.ci/scripts/check_release.py

@@ -6,7 +6,6 @@
 import tomllib
 import yaml
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from packaging.version import Version
 from git import Repo
 
@@ -25,6 +24,12 @@ def options():
         "'supported'. Defaults to 'supported', see `supported_release_branches` in "
         "`plugin_template.yml`.",
     )
+    parser.add_argument(
+        "--no-fetch",
+        default=False,
+        action="store_true",
+        help="Don't fetch remote. Run faster at the expense of maybe being outdated.",
+    )
     return parser.parse_args()
 
 
@@ -50,10 +55,15 @@ def current_version(repo, commitish):
 
 def check_pyproject_dependencies(repo, from_commit, to_commit):
     try:
-        old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml"))
+        new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml"))
+        try:
+            new_dependencies = set(new_pyproject["project"]["dependencies"])
+        except KeyError:
+            # New branch does not declare dependencies in pyproject.toml.
+            # Assume no release needed for this reason.
+            return []
+        old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml"))
         old_dependencies = set(old_pyproject["project"]["dependencies"])
-        new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml"))
-        new_dependencies = set(new_pyproject["project"]["dependencies"])
         if old_dependencies != new_dependencies:
             return ["dependencies"]
         else:
@@ -65,97 +75,100 @@ def check_pyproject_dependencies(repo, from_commit, to_commit):
 
 
 def main(options, template_config):
-    with TemporaryDirectory() as d:
-        # Clone from upstream to ensure we have updated branches & main
-        GITHUB_ORG = template_config["github_org"]
-        PLUGIN_NAME = template_config["plugin_name"]
-        UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git"
-        DEFAULT_BRANCH = template_config["plugin_default_branch"]
-
-        repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none")
-        heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")]
-        available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
-        available_branches.sort(key=lambda ver: Version(ver))
-        available_branches.append(DEFAULT_BRANCH)
-
-        branches = options.branches
-        if branches == "supported":
-            with open(f"{d}/template_config.yml", mode="r") as f:
-                tc = yaml.safe_load(f)
-                branches = set(tc["supported_release_branches"])
-            latest_release_branch = tc["latest_release_branch"]
-            if latest_release_branch is not None:
-                branches.add(latest_release_branch)
-            branches.add(DEFAULT_BRANCH)
-        else:
-            branches = set(branches.split(","))
-
-        if diff := branches - set(available_branches):
-            print(f"Supplied branches contains non-existent branches! {diff}")
-            exit(1)
-
-        print(f"Checking for releases on branches: {branches}")
-
-        releases = []
-        for branch in branches:
-            if branch != DEFAULT_BRANCH:
-                # Check if a Z release is needed
-                reasons = []
-                changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
-                z_changelog = False
-                for change in changes.split("\n"):
-                    # Check each changelog file to make sure everything checks out
-                    _, ext = os.path.splitext(change)
-                    if ext in Y_CHANGELOG_EXTS:
-                        print(
-                            f"Warning: A non-backported changelog ({change}) is present in the "
-                            f"{branch} release branch!"
-                        )
-                    elif ext in Z_CHANGELOG_EXTS:
-                        z_changelog = True
-                if z_changelog:
-                    reasons.append("Backports")
-
-                last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
-                req_txt_diff = repo.git.diff(
-                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
-                )
-                if req_txt_diff:
-                    reasons.append("requirements.txt")
-                pyproject_diff = repo.git.diff(
-                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml"
-                )
-                if pyproject_diff:
-                    reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}"))
-
-                if reasons:
-                    curr_version = Version(last_tag)
-                    assert curr_version.base_version.startswith(
-                        branch
-                    ), "Current-version has to belong to the current branch!"
-                    next_version = Version(f"{branch}.{curr_version.micro + 1}")
+    DEFAULT_BRANCH = template_config["plugin_default_branch"]
+
+    repo = Repo()
+
+    upstream_default_branch = next(
+        (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH)
+    ).tracking_branch()
+    remote = upstream_default_branch.remote_name
+    if not options.no_fetch:
+        repo.remote(remote).fetch()
+
+    # Warning: This will not work if branch names contain "/" but we don't really care here.
+    heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")]
+    available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+    available_branches.sort(key=lambda ver: Version(ver))
+    available_branches.append(DEFAULT_BRANCH)
+
+    branches = options.branches
+    if branches == "supported":
+        tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml"))
+        branches = set(tc["supported_release_branches"])
+        latest_release_branch = tc["latest_release_branch"]
+        if latest_release_branch is not None:
+            branches.add(latest_release_branch)
+        branches.add(DEFAULT_BRANCH)
+    else:
+        branches = set(branches.split(","))
+
+    if diff := branches - set(available_branches):
+        print(f"Supplied branches contains non-existent branches! {diff}")
+        exit(1)
+
+    print(f"Checking for releases on branches: {branches}")
+
+    releases = []
+    for branch in branches:
+        if branch != DEFAULT_BRANCH:
+            # Check if a Z release is needed
+            reasons = []
+            changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/")
+            z_changelog = False
+            for change in changes.split("\n"):
+                # Check each changelog file to make sure everything checks out
+                _, ext = os.path.splitext(change)
+                if ext in Y_CHANGELOG_EXTS:
                     print(
-                        f"A Z-release is needed for {branch}, "
-                        f"Prev: {last_tag}, "
-                        f"Next: {next_version.base_version}, "
-                        f"Reason: {','.join(reasons)}"
+                        f"Warning: A non-backported changelog ({change}) is present in the "
+                        f"{branch} release branch!"
                     )
+                elif ext in Z_CHANGELOG_EXTS:
+                    z_changelog = True
+            if z_changelog:
+                reasons.append("Backports")
+
+            last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}")
+            req_txt_diff = repo.git.diff(
+                f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt"
+            )
+            if req_txt_diff:
+                reasons.append("requirements.txt")
+            pyproject_diff = repo.git.diff(
+                f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml"
+            )
+            if pyproject_diff:
+                reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}"))
+
+            if reasons:
+                curr_version = Version(last_tag)
+                assert curr_version.base_version.startswith(
+                    branch
+                ), "Current-version has to belong to the current branch!"
+                next_version = Version(f"{branch}.{curr_version.micro + 1}")
+                print(
+                    f"A Z-release is needed for {branch}, "
+                    f"Prev: {last_tag}, "
+                    f"Next: {next_version.base_version}, "
+                    f"Reason: {','.join(reasons)}"
+                )
+                releases.append(next_version)
+        else:
+            # Check if a Y release is needed
+            changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/")
+            for change in changes.split("\n"):
+                _, ext = os.path.splitext(change)
+                if ext in Y_CHANGELOG_EXTS:
+                    # We don't put Y release bumps in the commit message, check file instead.
+                    # The 'current_version' is always the dev of the next version to release.
+                    next_version = current_version(repo, DEFAULT_BRANCH).base_version
+                    print(f"A new Y-release is needed! New Version: {next_version}")
                     releases.append(next_version)
-            else:
-                # Check if a Y release is needed
-                changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/")
-                for change in changes.split("\n"):
-                    _, ext = os.path.splitext(change)
-                    if ext in Y_CHANGELOG_EXTS:
-                        # We don't put Y release bumps in the commit message, check file instead.
-                        # The 'current_version' is always the dev of the next version to release.
-                        next_version = current_version(repo, DEFAULT_BRANCH).base_version
-                        print(f"A new Y-release is needed! New Version: {next_version}")
-                        releases.append(next_version)
-                        break
-
-        if len(releases) == 0:
-            print("No new releases to perform.")
+                    break
+
+    if len(releases) == 0:
+        print("No new releases to perform.")
 
 
 if __name__ == "__main__":

.ci/scripts/validate_commit_message.py

@@ -5,59 +5,78 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+import os
 import re
+import subprocess
 import sys
+import tomllib
 from pathlib import Path
-import subprocess
-import os
-import warnings
+
 from github import Github
 
-CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+with open("pyproject.toml", "rb") as fp:
+    PYPROJECT_TOML = tomllib.load(fp)
 KEYWORDS = ["fixes", "closes"]
+BLOCKING_REGEX = [
+    r"^DRAFT",
+    r"^WIP",
+    r"^NOMERGE",
+    r"^DO\s*NOT\s*MERGE",
+    r"^EXPERIMENT",
+    r"^FIXUP",
+    r"Apply suggestions from code review",
+]
+try:
+    CHANGELOG_EXTS = [
+        f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+    ]
+except KeyError:
+    CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"]
+NOISSUE_MARKER = "[noissue]"
 
 sha = sys.argv[1]
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
+if NOISSUE_MARKER in message:
+    sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.")
+
+if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)):
+    sys.exit("This PR is not ready for consumption.")
+
 g = Github(os.environ.get("GITHUB_TOKEN"))
 repo = g.get_repo("pulp/pulp_python")
 
 
-def __check_status(issue):
+def check_status(issue):
     gi = repo.get_issue(int(issue))
     if gi.pull_request:
         sys.exit(f"Error: issue #{issue} is a pull request.")
-    if gi.closed_at and "cherry picked from commit" not in message:
-        warnings.warn(
-            "When backporting, use the -x flag to append a line that says "
-            "'(cherry picked from commit ...)' to the original commit message."
-        )
+    if gi.closed_at:
         sys.exit(f"Error: issue #{issue} is closed.")
 
 
-def __check_changelog(issue):
+def check_changelog(issue):
     matches = list(Path("CHANGES").rglob(f"{issue}.*"))
 
     if len(matches) < 1:
         sys.exit(f"Could not find changelog entry in CHANGES/ for {issue}.")
     for match in matches:
         if match.suffix not in CHANGELOG_EXTS:
             sys.exit(f"Invalid extension for changelog entry '{match}'.")
-        if match.suffix == ".feature" and "cherry picked from commit" in message:
-            sys.exit(f"Can not backport '{match}' as it is a feature.")
 
 
 print("Checking commit message for {sha}.".format(sha=sha[0:7]))
 
 # validate the issue attached to the commit
-regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
-pattern = re.compile(regex, re.IGNORECASE)
-
-issues = pattern.findall(message)
+issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
+issues = re.findall(issue_regex, message, re.IGNORECASE)
+cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$"
+cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE)
 
 if issues:
-    for issue in pattern.findall(message):
-        __check_status(issue)
-        __check_changelog(issue)
+    for issue in issues:
+        if not cherry_pick:
+            check_status(issue)
+            check_changelog(issue)
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-406-g5f397e3
+2021.08.26-417-gcec8e48

.github/workflows/pr_checks.yml

@@ -9,7 +9,13 @@
 name: "Python PR static checks"
 on:
   pull_request_target:
-    types: ["opened", "synchronize", "reopened"]
+    types:
+      - "opened"
+      - "synchronize"
+      - "reopened"
+    branches:
+      - "main"
+      - "[0-9]+.[0-9]+"
 
 # This workflow runs with elevated permissions.
 # Do not even think about running a single bit of code from the PR.

.github/workflows/scripts/check_commit.sh

@@ -15,8 +15,4 @@ set -euv
 for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha')
 do
   python3 .ci/scripts/validate_commit_message.py "$SHA"
-  VALUE=$?
-  if [ "$VALUE" -gt 0 ]; then
-    exit $VALUE
-  fi
 done

.github/workflows/scripts/publish_client_gem.sh

@@ -14,22 +14,15 @@ cd "$(dirname "$(realpath -e "$0")")"/../../..
 
 VERSION="$1"
 
-if [[ -z "$VERSION" ]]; then
+if [[ -z "${VERSION}" ]]
+then
   echo "No version specified."
   exit 1
 fi
 
-RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_python_client/versions/$VERSION")"
-
-if [ "$RESPONSE" == "200" ];
-then
-  echo "pulp_python client $VERSION has already been released. Skipping."
-  exit
-fi
-
 mkdir -p ~/.gem
 touch ~/.gem/credentials
 echo "---
-:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials
+:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials
 sudo chmod 600 ~/.gem/credentials
 gem push "pulp_python_client-${VERSION}.gem"