From 80d4231b6b0cbad1a28c31049748f478864c4385 Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 12 May 2023 16:12:04 -0700 Subject: [PATCH 1/3] Update relnotes scripts. 1. Update relnotes.sh to use relnotes.py to generate release notes if not a rolling release 2. Update relnotes.py: - Include co-authors in acknowledgements - Make sure older releases work with the correct "last" release - For minor/patch release, use commit message if no RELNOTES found - Add option to sort relnotes by category - Other minor fixes/updates PiperOrigin-RevId: 531628752 Change-Id: Ifdcf55d319e7221b7ed4eb7e510a3c4c9a88b41d --- scripts/release/relnotes.py | 157 +++++++++++++++++++++++++++++++----- scripts/release/relnotes.sh | 68 ++++++++-------- 2 files changed, 170 insertions(+), 55 deletions(-) diff --git a/scripts/release/relnotes.py b/scripts/release/relnotes.py index 792816785435fc..15feff24ada134 100644 --- a/scripts/release/relnotes.py +++ b/scripts/release/relnotes.py @@ -14,25 +14,20 @@ """Script to generate release notes.""" +import os import re import subprocess - +import sys import requests -def get_last_release(): - """Discovers the last stable release name from GitHub.""" - response = requests.get("https://github.com/bazelbuild/bazel/releases/latest") - return response.url.split("/")[-1] - - def git(*args): """Runs git as a subprocess, and returns its stdout as a list of lines.""" return subprocess.check_output(["git"] + list(args)).decode("utf-8").strip().split("\n") -def extract_relnotes(commit_message_lines): +def extract_relnotes(commit_message_lines, is_major_release): """Extracts relnotes from a commit message (passed in as a list of lines).""" relnote_lines = [] in_relnote = False @@ -50,14 +45,27 @@ def extract_relnotes(commit_message_lines): relnote_lines.append(line) relnote = " ".join(relnote_lines) relnote_lower = relnote.strip().lower().rstrip(".") - if relnote_lower == "n/a" or relnote_lower == "none": - return None + if relnote_lower == "n/a" or relnote_lower == "none" or not relnote_lower: + if is_major_release: + return None + relnote = re.sub( + r"\[\d+\.\d+\.\d\]\s?", "", commit_message_lines[0].strip() + ) + else: + issue_id = re.search( + r"\(\#[0-9]+\)$", commit_message_lines[0].strip().split()[-1] + ) + if issue_id: + relnote = relnote + " " + issue_id.group(0).strip() + return relnote -def get_relnotes_between(base, head): +def get_relnotes_between(base, head, is_major_release): """Gets all relnotes for commits between `base` and `head`.""" - commits = git("rev-list", f"{base}..{head}", "--grep=RELNOTES") + commits = git("rev-list", f"{base}..{head}") + if commits == [""]: + return [] relnotes = [] rolled_back_commits = set() # We go in reverse-chronological order, so that we can identify rollback @@ -71,28 +79,117 @@ def get_relnotes_between(base, head): rolled_back_commits.add(m[1]) # The rollback commit itself is also skipped. continue - relnote = extract_relnotes(lines) + relnote = extract_relnotes(lines, is_major_release) if relnote is not None: relnotes.append(relnote) return relnotes +def get_label(issue_id): + """Get team-X label added to issue.""" + auth = os.system( + "gsutil cat" + " gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc |" + " gcloud kms decrypt --project bazel-public --location global" + " --keyring buildkite --key github-trusted-token --ciphertext-file" + " - --plaintext-file -" + ) + headers = { + "Authorization": "Bearer " + auth, + "Accept": "application/vnd.github+json", + } + response = requests.get( + "https://api.github.com/repos/bazelbuild/bazel/issues/" + + issue_id + "/labels", headers=headers, + ) + for item in response.json(): + for key, value in item.items(): + if key == "name" and "team-" in value: + return value.strip() + return None + + +def get_categorized_relnotes(filtered_notes): + """Sort release notes by category.""" + categorized_relnotes = {} + for relnote in filtered_notes: + issue_id = re.search(r"\(\#[0-9]+\)$", relnote.strip().split()[-1]) + category = None + if issue_id: + category = get_label(re.sub(r"\(|\#|\)", "", issue_id.group(0).strip())) + + if category is None: + category = "General" + else: + category = re.sub("team-", "", category) + + try: + categorized_relnotes[category].append(relnote) + except KeyError: + categorized_relnotes[category] = [relnote] + + return dict(sorted(categorized_relnotes.items())) + + +def get_external_authors_between(base, head): + """Gets all external authors for commits between `base` and `head`.""" + + # Get all authors + authors = git("log", f"{base}..{head}", "--format=%aN|%aE") + authors = set( + author.partition("|")[0].rstrip() + for author in authors + if not (author.endswith(("@google.com", "@users.noreply.github.com"))) + ) + + # Get all co-authors + contributors = git( + "log", f"{base}..{head}", "--format=%(trailers:key=Co-authored-by)" + ) + + coauthors = [] + for coauthor in contributors: + if coauthor and not re.search( + "@google.com|@users.noreply.github.com", coauthor + ): + coauthors.append( + " ".join(re.sub(r"Co-authored-by: |<.*?>", "", coauthor).split()) + ) + return ", ".join(sorted(authors.union(coauthors), key=str.casefold)) + + if __name__ == "__main__": - # Get the last stable release. - last_release = get_last_release() - print("last_release is", last_release) - git("fetch", "origin", f"refs/tags/{last_release}:refs/tags/{last_release}") + # Get last release and make sure it's consistent with current X.Y.Z release + # e.g. if current_release is 5.3.3, last_release should be 5.3.2 even if + # latest release is 6.1.1 + current_release = git("rev-parse", "--abbrev-ref", "HEAD") + current_release = re.sub( + r"rc\d", "", current_release[0].removeprefix("release-") + ) + + is_major = bool(re.fullmatch(r"\d+.0.0", current_release)) + + tags = [tag for tag in git("tag", "--sort=refname") if "pre" not in tag] + if current_release not in tags: + tags.append(current_release) + tags.sort() + last_release = tags[tags.index(current_release) - 1] + else: + print("Error: release tag already exists") + sys.exit(1) # Assuming HEAD is on the current (to-be-released) release, find the merge # base with the last release so that we know which commits to generate notes # for. merge_base = git("merge-base", "HEAD", last_release)[0] - print("merge base with", last_release, "is", merge_base) + print("Baseline: ", merge_base) # Generate notes for all commits from last branch cut to HEAD, but filter out # any identical notes from the previous release branch. - cur_release_relnotes = get_relnotes_between(merge_base, "HEAD") - last_release_relnotes = set(get_relnotes_between(merge_base, last_release)) + cur_release_relnotes = get_relnotes_between(merge_base, "HEAD", is_major) + last_release_relnotes = set( + get_relnotes_between(merge_base, last_release, is_major) + ) filtered_relnotes = [ note for note in cur_release_relnotes if note not in last_release_relnotes ] @@ -100,6 +197,22 @@ def get_relnotes_between(base, head): # Reverse so that the notes are in chronological order. filtered_relnotes.reverse() print() + print("Release Notes:") + + if len(sys.argv) >= 2 and sys.argv[1] == "sort": + print() + categorized_release_notes = get_categorized_relnotes(filtered_relnotes) + for label in categorized_release_notes: + print(label + ":") + for note in categorized_release_notes[label]: + print("+", note) + print() + else: + for note in filtered_relnotes: + print("+", note) + print() - for note in filtered_relnotes: - print("*", note) + print("Acknowledgements:") + external_authors = get_external_authors_between(merge_base, "HEAD") + print("This release contains contributions from many people at Google" + + ("." if not external_authors else f", as well as {external_authors}.")) diff --git a/scripts/release/relnotes.sh b/scripts/release/relnotes.sh index 8e2986affcde82..9ad785a752eb49 100755 --- a/scripts/release/relnotes.sh +++ b/scripts/release/relnotes.sh @@ -270,50 +270,52 @@ function generate_release_message() { local release_name="$1" local branch="${2:-HEAD}" local delimiter="${3-}" - local baseline="$(get_release_baseline "${branch}")" - local cherrypicks="$(get_cherrypicks "${branch}" "${baseline}")" get_release_title "$release_name" echo - if [ -n "${delimiter}" ]; then - echo "${delimiter}" - fi - __create_revision_information $baseline $cherrypicks - if [ -n "${delimiter}" ]; then - echo "${delimiter}" - fi - - echo + if [[ "$(is_rolling_release)" -eq 0 ]]; then + if [ -n "${delimiter}" ]; then + echo "${delimiter}" + fi + python3 ${RELNOTES_SCRIPT_DIR}/relnotes.py + if [ -n "${delimiter}" ]; then + echo "${delimiter}" + fi + else + local baseline="$(get_release_baseline "${branch}")" + local cherrypicks="$(get_cherrypicks "${branch}" "${baseline}")" - # Generate the release notes - local tmpfile=$(mktemp --tmpdir relnotes-XXXXXXXX) - trap "rm -f ${tmpfile}" EXIT + if [ -n "${delimiter}" ]; then + echo "${delimiter}" + fi + __create_revision_information $baseline $cherrypicks + if [ -n "${delimiter}" ]; then + echo "${delimiter}" + fi - # Save the changelog so we compute the relnotes against HEAD. - git show master:CHANGELOG.md > "${tmpfile}" + echo - local relnotes="$(create_release_notes "${tmpfile}" "${baseline}" ${cherrypicks})" - echo "${relnotes}" > "${tmpfile}" + # Generate the release notes + local tmpfile=$(mktemp --tmpdir relnotes-XXXXXXXX) + trap "rm -f ${tmpfile}" EXIT - __release_note_processor "${tmpfile}" || return 1 - relnotes="$(cat ${tmpfile})" + # Save the changelog so we compute the relnotes against HEAD. + git show master:CHANGELOG.md > "${tmpfile}" - cat "${tmpfile}" -} + local relnotes="$(create_release_notes "${tmpfile}" "${baseline}" ${cherrypicks})" + echo "${relnotes}" > "${tmpfile}" -# Returns the release notes for the CHANGELOG.md taken from either from -# the notes for a release candidate/rolling release, or from the commit message for a -# full release. -function get_full_release_notes() { - local release_name="$(get_full_release_name "$@")" + __release_note_processor "${tmpfile}" || return 1 + relnotes="$(cat ${tmpfile})" - if [[ "${release_name}" =~ rc[0-9]+$ ]] || [[ "$(is_rolling_release)" -eq 1 ]]; then - # Release candidate or rolling release -> generate from the notes - generate_release_message "${release_name}" "$@" - else - # Full LTS release -> return the commit message - git_commit_msg "$@" + cat "${tmpfile}" fi } +# Returns the release notes for the CHANGELOG.md for all releases - +# release candidate, full release, and rolling release. +function get_full_release_notes() { + local release_name="$(get_full_release_name "$@")" + generate_release_message "${release_name}" "$@" +} \ No newline at end of file From 63e0dbcedf95205ca5a9f0bbda662a6ac5adfe80 Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 22 May 2023 09:34:24 -0700 Subject: [PATCH 2/3] Updates/fixes to relnotes script - Replace removeprefix since this requires python 3.9 - Include authors/co-authors with @users.noreply.github.com in acknowledgement section (we're currently not adding contributors w/o public emails) PiperOrigin-RevId: 534091486 Change-Id: I455f96310062069148d2a28758e65f9f5675d3ff --- scripts/release/relnotes.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/release/relnotes.py b/scripts/release/relnotes.py index 15feff24ada134..66581cff627514 100644 --- a/scripts/release/relnotes.py +++ b/scripts/release/relnotes.py @@ -138,9 +138,7 @@ def get_external_authors_between(base, head): authors = git("log", f"{base}..{head}", "--format=%aN|%aE") authors = set( author.partition("|")[0].rstrip() - for author in authors - if not (author.endswith(("@google.com", "@users.noreply.github.com"))) - ) + for author in authors if not (author.endswith(("@google.com")))) # Get all co-authors contributors = git( @@ -149,9 +147,7 @@ def get_external_authors_between(base, head): coauthors = [] for coauthor in contributors: - if coauthor and not re.search( - "@google.com|@users.noreply.github.com", coauthor - ): + if coauthor and not re.search("@google.com", coauthor): coauthors.append( " ".join(re.sub(r"Co-authored-by: |<.*?>", "", coauthor).split()) ) @@ -162,11 +158,12 @@ def get_external_authors_between(base, head): # Get last release and make sure it's consistent with current X.Y.Z release # e.g. if current_release is 5.3.3, last_release should be 5.3.2 even if # latest release is 6.1.1 - current_release = git("rev-parse", "--abbrev-ref", "HEAD") - current_release = re.sub( - r"rc\d", "", current_release[0].removeprefix("release-") - ) + current_release = git("rev-parse", "--abbrev-ref", "HEAD")[0] + if not current_release.startswith("release-"): + print("Error: Not a release- branch") + sys.exit(1) + current_release = re.sub(r"rc\d", "", current_release[len("release-"):]) is_major = bool(re.fullmatch(r"\d+.0.0", current_release)) tags = [tag for tag in git("tag", "--sort=refname") if "pre" not in tag] From 9e0a12bafa948adcf18b0865116d5c540d361731 Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 24 May 2023 08:31:31 -0700 Subject: [PATCH 3/3] Fix relnotes script - Update GitHub token retrieval - Allow checking out tags (final releases) in addition to branches (RCs) PiperOrigin-RevId: 534851155 Change-Id: I8b92898a8f5dd20139ca2467bb8604c6a8eabd67 --- scripts/release/relnotes.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/release/relnotes.py b/scripts/release/relnotes.py index 66581cff627514..402475a2932be4 100644 --- a/scripts/release/relnotes.py +++ b/scripts/release/relnotes.py @@ -14,7 +14,6 @@ """Script to generate release notes.""" -import os import re import subprocess import sys @@ -87,13 +86,13 @@ def get_relnotes_between(base, head, is_major_release): def get_label(issue_id): """Get team-X label added to issue.""" - auth = os.system( + auth = subprocess.check_output( "gsutil cat" " gs://bazel-trusted-encrypted-secrets/github-trusted-token.enc |" " gcloud kms decrypt --project bazel-public --location global" " --keyring buildkite --key github-trusted-token --ciphertext-file" - " - --plaintext-file -" - ) + " - --plaintext-file -", shell=True + ).decode("utf-8").strip().split("\n")[0] headers = { "Authorization": "Bearer " + auth, "Accept": "application/vnd.github+json", @@ -159,11 +158,16 @@ def get_external_authors_between(base, head): # e.g. if current_release is 5.3.3, last_release should be 5.3.2 even if # latest release is 6.1.1 current_release = git("rev-parse", "--abbrev-ref", "HEAD")[0] - if not current_release.startswith("release-"): - print("Error: Not a release- branch") - sys.exit(1) - current_release = re.sub(r"rc\d", "", current_release[len("release-"):]) + if current_release.startswith("release-"): + current_release = re.sub(r"rc\d", "", current_release[len("release-"):]) + else: + try: + current_release = git("describe", "--tags")[0] + except Exception: # pylint: disable=broad-exception-caught + print("Error: Not a release branch.") + sys.exit(1) + is_major = bool(re.fullmatch(r"\d+.0.0", current_release)) tags = [tag for tag in git("tag", "--sort=refname") if "pre" not in tag]