From 7a0e5358db5e76a588b586892a2a50eac134edac Mon Sep 17 00:00:00 2001 From: pulpbot Date: Mon, 16 Dec 2024 10:41:04 +0000 Subject: [PATCH] Update CI files --- .ci/scripts/check_release.py | 197 ++++++++++-------- .ci/scripts/validate_commit_message.py | 59 ++++-- .github/template_gitref | 2 +- .github/workflows/pr_checks.yml | 8 +- .github/workflows/scripts/check_commit.sh | 4 - .../workflows/scripts/publish_client_gem.sh | 13 +- .../workflows/scripts/publish_client_pypi.sh | 17 +- .../workflows/scripts/publish_plugin_pypi.sh | 17 +- .github/workflows/update_ci.yml | 9 + 9 files changed, 175 insertions(+), 151 deletions(-) diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index da45be6a..dfc4a41e 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -6,7 +6,6 @@ import tomllib import yaml from pathlib import Path -from tempfile import TemporaryDirectory from packaging.version import Version from git import Repo @@ -25,6 +24,12 @@ def options(): "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) + parser.add_argument( + "--no-fetch", + default=False, + action="store_true", + help="Don't fetch remote. Run faster at the expense of maybe being outdated.", + ) return parser.parse_args() @@ -50,10 +55,15 @@ def current_version(repo, commitish): def check_pyproject_dependencies(repo, from_commit, to_commit): try: - old_pyproject = tomllib.load(repo.show("{from_commit}:pyproject.toml")) + new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml")) + try: + new_dependencies = set(new_pyproject["project"]["dependencies"]) + except KeyError: + # New branch does not declare dependencies in pyproject.toml. + # Assume no release needed for this reason. + return [] + old_pyproject = tomllib.loads(repo.git.show(f"{from_commit}:pyproject.toml")) old_dependencies = set(old_pyproject["project"]["dependencies"]) - new_pyproject = tomllib.load(repo.show("{to_commit}:pyproject.toml")) - new_dependencies = set(new_pyproject["project"]["dependencies"]) if old_dependencies != new_dependencies: return ["dependencies"] else: @@ -65,97 +75,100 @@ def check_pyproject_dependencies(repo, from_commit, to_commit): def main(options, template_config): - with TemporaryDirectory() as d: - # Clone from upstream to ensure we have updated branches & main - GITHUB_ORG = template_config["github_org"] - PLUGIN_NAME = template_config["plugin_name"] - UPSTREAM_REMOTE = f"https://github.com/{GITHUB_ORG}/{PLUGIN_NAME}.git" - DEFAULT_BRANCH = template_config["plugin_default_branch"] - - repo = Repo.clone_from(UPSTREAM_REMOTE, d, filter="blob:none") - heads = [h.split("/")[-1] for h in repo.git.ls_remote("--heads").split("\n")] - available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] - available_branches.sort(key=lambda ver: Version(ver)) - available_branches.append(DEFAULT_BRANCH) - - branches = options.branches - if branches == "supported": - with open(f"{d}/template_config.yml", mode="r") as f: - tc = yaml.safe_load(f) - branches = set(tc["supported_release_branches"]) - latest_release_branch = tc["latest_release_branch"] - if latest_release_branch is not None: - branches.add(latest_release_branch) - branches.add(DEFAULT_BRANCH) - else: - branches = set(branches.split(",")) - - if diff := branches - set(available_branches): - print(f"Supplied branches contains non-existent branches! {diff}") - exit(1) - - print(f"Checking for releases on branches: {branches}") - - releases = [] - for branch in branches: - if branch != DEFAULT_BRANCH: - # Check if a Z release is needed - reasons = [] - changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") - z_changelog = False - for change in changes.split("\n"): - # Check each changelog file to make sure everything checks out - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - print( - f"Warning: A non-backported changelog ({change}) is present in the " - f"{branch} release branch!" - ) - elif ext in Z_CHANGELOG_EXTS: - z_changelog = True - if z_changelog: - reasons.append("Backports") - - last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") - req_txt_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" - ) - if req_txt_diff: - reasons.append("requirements.txt") - pyproject_diff = repo.git.diff( - f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" - ) - if pyproject_diff: - reasons.extend(check_pyproject_dependencies(repo, last_tag, f"origin/{branch}")) - - if reasons: - curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" - next_version = Version(f"{branch}.{curr_version.micro + 1}") + DEFAULT_BRANCH = template_config["plugin_default_branch"] + + repo = Repo() + + upstream_default_branch = next( + (branch for branch in repo.branches if branch.name == DEFAULT_BRANCH) + ).tracking_branch() + remote = upstream_default_branch.remote_name + if not options.no_fetch: + repo.remote(remote).fetch() + + # Warning: This will not work if branch names contain "/" but we don't really care here. + heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")] + available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)] + available_branches.sort(key=lambda ver: Version(ver)) + available_branches.append(DEFAULT_BRANCH) + + branches = options.branches + if branches == "supported": + tc = yaml.safe_load(repo.git.show(f"{upstream_default_branch}:template_config.yml")) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) + else: + branches = set(branches.split(",")) + + if diff := branches - set(available_branches): + print(f"Supplied branches contains non-existent branches! {diff}") + exit(1) + + print(f"Checking for releases on branches: {branches}") + + releases = [] + for branch in branches: + if branch != DEFAULT_BRANCH: + # Check if a Z release is needed + reasons = [] + changes = repo.git.ls_tree("-r", "--name-only", f"{remote}/{branch}", "CHANGES/") + z_changelog = False + for change in changes.split("\n"): + # Check each changelog file to make sure everything checks out + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: print( - f"A Z-release is needed for {branch}, " - f"Prev: {last_tag}, " - f"Next: {next_version.base_version}, " - f"Reason: {','.join(reasons)}" + f"Warning: A non-backported changelog ({change}) is present in the " + f"{branch} release branch!" ) + elif ext in Z_CHANGELOG_EXTS: + z_changelog = True + if z_changelog: + reasons.append("Backports") + + last_tag = repo.git.describe("--tags", "--abbrev=0", f"{remote}/{branch}") + req_txt_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "requirements.txt" + ) + if req_txt_diff: + reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"{remote}/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.extend(check_pyproject_dependencies(repo, last_tag, f"{remote}/{branch}")) + + if reasons: + curr_version = Version(last_tag) + assert curr_version.base_version.startswith( + branch + ), "Current-version has to belong to the current branch!" + next_version = Version(f"{branch}.{curr_version.micro + 1}") + print( + f"A Z-release is needed for {branch}, " + f"Prev: {last_tag}, " + f"Next: {next_version.base_version}, " + f"Reason: {','.join(reasons)}" + ) + releases.append(next_version) + else: + # Check if a Y release is needed + changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") + for change in changes.split("\n"): + _, ext = os.path.splitext(change) + if ext in Y_CHANGELOG_EXTS: + # We don't put Y release bumps in the commit message, check file instead. + # The 'current_version' is always the dev of the next version to release. + next_version = current_version(repo, DEFAULT_BRANCH).base_version + print(f"A new Y-release is needed! New Version: {next_version}") releases.append(next_version) - else: - # Check if a Y release is needed - changes = repo.git.ls_tree("-r", "--name-only", DEFAULT_BRANCH, "CHANGES/") - for change in changes.split("\n"): - _, ext = os.path.splitext(change) - if ext in Y_CHANGELOG_EXTS: - # We don't put Y release bumps in the commit message, check file instead. - # The 'current_version' is always the dev of the next version to release. - next_version = current_version(repo, DEFAULT_BRANCH).base_version - print(f"A new Y-release is needed! New Version: {next_version}") - releases.append(next_version) - break - - if len(releases) == 0: - print("No new releases to perform.") + break + + if len(releases) == 0: + print("No new releases to perform.") if __name__ == "__main__": diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 739f75c5..04e3fd30 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,37 +5,57 @@ # # For more info visit https://github.com/pulp/plugin_template +import os import re +import subprocess import sys +import tomllib from pathlib import Path -import subprocess -import os -import warnings + from github import Github -CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] +with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) KEYWORDS = ["fixes", "closes"] +BLOCKING_REGEX = [ + r"^DRAFT", + r"^WIP", + r"^NOMERGE", + r"^DO\s*NOT\s*MERGE", + r"^EXPERIMENT", + r"^FIXUP", + r"Apply suggestions from code review", +] +try: + CHANGELOG_EXTS = [ + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + ] +except KeyError: + CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"] +NOISSUE_MARKER = "[noissue]" sha = sys.argv[1] message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8") +if NOISSUE_MARKER in message: + sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.") + +if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)): + sys.exit("This PR is not ready for consumption.") + g = Github(os.environ.get("GITHUB_TOKEN")) repo = g.get_repo("pulp/pulp_python") -def __check_status(issue): +def check_status(issue): gi = repo.get_issue(int(issue)) if gi.pull_request: sys.exit(f"Error: issue #{issue} is a pull request.") - if gi.closed_at and "cherry picked from commit" not in message: - warnings.warn( - "When backporting, use the -x flag to append a line that says " - "'(cherry picked from commit ...)' to the original commit message." - ) + if gi.closed_at: sys.exit(f"Error: issue #{issue} is closed.") -def __check_changelog(issue): +def check_changelog(issue): matches = list(Path("CHANGES").rglob(f"{issue}.*")) if len(matches) < 1: @@ -43,21 +63,20 @@ def __check_changelog(issue): for match in matches: if match.suffix not in CHANGELOG_EXTS: sys.exit(f"Invalid extension for changelog entry '{match}'.") - if match.suffix == ".feature" and "cherry picked from commit" in message: - sys.exit(f"Can not backport '{match}' as it is a feature.") print("Checking commit message for {sha}.".format(sha=sha[0:7])) # validate the issue attached to the commit -regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) -pattern = re.compile(regex, re.IGNORECASE) - -issues = pattern.findall(message) +issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS)) +issues = re.findall(issue_regex, message, re.IGNORECASE) +cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$" +cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE) if issues: - for issue in pattern.findall(message): - __check_status(issue) - __check_changelog(issue) + for issue in issues: + if not cherry_pick: + check_status(issue) + check_changelog(issue) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 22e8d0a9..51006453 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-406-g5f397e3 +2021.08.26-417-gcec8e48 diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index c4898097..0e0a7936 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -9,7 +9,13 @@ name: "Python PR static checks" on: pull_request_target: - types: ["opened", "synchronize", "reopened"] + types: + - "opened" + - "synchronize" + - "reopened" + branches: + - "main" + - "[0-9]+.[0-9]+" # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. diff --git a/.github/workflows/scripts/check_commit.sh b/.github/workflows/scripts/check_commit.sh index e6b6d5e3..400e0542 100755 --- a/.github/workflows/scripts/check_commit.sh +++ b/.github/workflows/scripts/check_commit.sh @@ -15,8 +15,4 @@ set -euv for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha') do python3 .ci/scripts/validate_commit_message.py "$SHA" - VALUE=$? - if [ "$VALUE" -gt 0 ]; then - exit $VALUE - fi done diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index bd367010..e771b5e7 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -14,22 +14,15 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulp_python_client/versions/$VERSION")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." - exit -fi - mkdir -p ~/.gem touch ~/.gem/credentials echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +:rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials sudo chmod 600 ~/.gem/credentials gem push "pulp_python_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index 43a31a35..144f2597 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -14,18 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")/../../.." VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python-client/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python client $VERSION has already been released. Skipping." -else - twine upload -u __token__ -p "$PYPI_API_TOKEN" \ - "dist/pulp_python_client-$VERSION-py3-none-any.whl" \ - "dist/pulp_python-client-$VERSION.tar.gz" -fi +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +"dist/pulp_python_client-${VERSION}-py3-none-any.whl" \ +"dist/pulp_python-client-${VERSION}.tar.gz" \ +; diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index 384cecb2..16b0ce05 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -14,20 +14,13 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. VERSION="$1" -if [[ -z "$VERSION" ]]; then +if [[ -z "${VERSION}" ]] +then echo "No version specified." exit 1 fi -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulp-python/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "pulp_python $VERSION has already been released. Skipping." - exit -fi - -twine upload -u __token__ -p "$PYPI_API_TOKEN" \ -dist/pulp?python-"$VERSION"-py3-none-any.whl \ -dist/pulp?python-"$VERSION".tar.gz \ +twine upload -u __token__ -p "${PYPI_API_TOKEN}" \ +dist/pulp?python-"${VERSION}"-py3-none-any.whl \ +dist/pulp?python-"${VERSION}".tar.gz \ ; diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 02f251fa..05b93764 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -56,6 +56,7 @@ jobs: - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" + id: "create_pr_main" with: token: "${{ secrets.RELEASE_TOKEN }}" path: "pulp_python" @@ -65,4 +66,12 @@ jobs: branch: "update-ci/main" base: "main" delete-branch: true + - name: "Mark PR automerge" + working-directory: "pulp_python" + run: | + gh pr merge --rebase --auto "${{ steps.create_pr_main.outputs.pull-request-number }}" + if: "steps.create_pr_main.outputs.pull-request-number" + env: + GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}" + continue-on-error: true ...