From 738443b944a95f6232d1db93970fe22241c9c212 Mon Sep 17 00:00:00 2001 From: brainbot-devops Date: Mon, 19 Aug 2019 10:08:59 +0200 Subject: [PATCH 1/4] Parse tags since last master tag. --- .circleci/scripts/update-changelog.py | 53 ++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/.circleci/scripts/update-changelog.py b/.circleci/scripts/update-changelog.py index 6b92746c7..07d9223e0 100644 --- a/.circleci/scripts/update-changelog.py +++ b/.circleci/scripts/update-changelog.py @@ -1,39 +1,80 @@ """TODO: This script is a stub.""" +import re +import subprocess + from typing import List, Tuple -from constants import PROJECT_GIT_DIR, CURRENT_BRANCH + +from constants import PROJECT_GIT_DIR, CURRENT_BRANCH, COMMIT_PATTERN if CURRENT_BRANCH != "master": exit(0) -def read_git_commit_history_since_tag(tag=None) -> Tuple[List[str], List[str], List[str]]: +def latest_tag(): + proc =subprocess.run( + f"git --git-dir={PROJECT_GIT_DIR} tag -l".split(" "), + check=True, + stdout=subprocess.PIPE, + ) + + all_tags = proc.stdout.decode("UTF-8").split("\n") + releases = sorted([tag for tag in all_tags if 'dev' not in tag and tag.startswith("v")], reverse=True) + latest, *_ = releases + return latest + + +def read_git_commit_history_since_tag(tag) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]], List[Tuple[str, str]]]: """Return a list of all git commit titles since the given `tag`. If `tag` is not given, we'll use the previous tag and compare the log up up to the current tag. The commits are returned as three lists: - feature commits + 1. feature commits 2. bugfix commits 3. hotfix commits """ completed_process = subprocess.run( - f"git --git-dir={PROJECT_GIT_DIR} log master..release --format=%s".split(" "), + f"git --git-dir={PROJECT_GIT_DIR} log master..{tag} --format=%s".split(" "), check=True, stdout=subprocess.PIPE, ) titles = completed_process.stdout.decode("UTF-8").split("\n") completed_process = subprocess.run( - f"git --git-dir={PROJECT_GIT_DIR} log master..release --format=%b".split(" "), + f"git --git-dir={PROJECT_GIT_DIR} log master..{tag} --format=%b".split(" "), check=True, stdout=subprocess.PIPE, ) bodies = completed_process.stdout.decode("UTF-8").split("\n") + assert len(titles) == len(bodies) + + pattern = re.compile(COMMIT_PATTERN) + + feats, fixes, hotfixes = [], [], [] + + for title, body in zip(titles, bodies): + match = pattern.match(title) + if not match: + continue + + commit_type = match.groupdict()["TYPE"] + + if commit_type == "FEAT": + feats.append((title, body)) + elif commit_type == "FIX": + fixes.append((title, body)) + elif commit_type == "HOTFIX": + hotfixes.append((title, body)) + else: + print(f"No type found, skipping commit '{title}'..") + + return feats, fixes, hotfixes + -def format_commits(*commits) -> List[str]: +def format_commits(*commits: List[Tuple[str, str]]) -> List[str]: """Format the given commits for writing to the Changelog. The expected input str format is: From 02efd0caade524e7caf0423b36d8a098a3d29c43 Mon Sep 17 00:00:00 2001 From: brainbot-devops Date: Thu, 22 Aug 2019 10:27:35 +0200 Subject: [PATCH 2/4] Create ChangeLog on Merge commits to master. --- .circleci/config.yml | 2 +- .circleci/scripts/constants.py | 2 +- .circleci/scripts/update-changelog.py | 78 ++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4286abde9..1ff112488 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -236,7 +236,7 @@ jobs: - run: name: Update Changelog entries command: | - python3 ${CI_SCRIPTS_DIR}/update-changelog.py + python3 ${CI_SCRIPTS_DIR}/update-changelog.py {PROJECT_ROOT}/CHANGELOG.rst # Publish a wheel and tarball of the Scenario Player to pypi. deploy-to-pypi: diff --git a/.circleci/scripts/constants.py b/.circleci/scripts/constants.py index 342c2bfca..30e6893d5 100644 --- a/.circleci/scripts/constants.py +++ b/.circleci/scripts/constants.py @@ -9,7 +9,7 @@ PROJECT_ROOT = os.environ["PROJECT_ROOT"] COMMIT_SHA = os.environ["CIRCLE_SHA1"] -COMMIT_PATTERN = r"^\[(?P(FEAT|FIX|HOTFIX))-(?P#\d+)\]\w?(?P)" +COMMIT_PATTERN = r"^\[(?P(FEAT|FIX|HOTFIX))-(?P#\d+)\]\w?(?P.*$)" RELEASE_COMMIT_PATTERN = r"^\[(?PRELEASE)\]\w?(?P)" BUMPVERSION_PREFIX = "Cut New Release:" CURRENT_BRANCH = os.environ.get("CIRCLE_BRANCH") diff --git a/.circleci/scripts/update-changelog.py b/.circleci/scripts/update-changelog.py index 07d9223e0..399684650 100644 --- a/.circleci/scripts/update-changelog.py +++ b/.circleci/scripts/update-changelog.py @@ -1,9 +1,9 @@ """TODO: This script is a stub.""" +import pathlib import re import subprocess -from typing import List, Tuple - +from typing import List, Set, Tuple from constants import PROJECT_GIT_DIR, CURRENT_BRANCH, COMMIT_PATTERN @@ -36,18 +36,21 @@ def read_git_commit_history_since_tag(tag) -> Tuple[List[Tuple[str, str]], List[ 3. hotfix commits """ completed_process = subprocess.run( - f"git --git-dir={PROJECT_GIT_DIR} log master..{tag} --format=%s".split(" "), + f"git --git-dir={PROJECT_GIT_DIR} log {tag}..master --format=%s".split(" "), check=True, stdout=subprocess.PIPE, ) titles = completed_process.stdout.decode("UTF-8").split("\n") + # The body of a commit may include newline characters, so we need to specify + # a custom separator to indicate the end of the commit body. + separator = "<><><>" completed_process = subprocess.run( - f"git --git-dir={PROJECT_GIT_DIR} log master..{tag} --format=%b".split(" "), + f"git --git-dir={PROJECT_GIT_DIR} log {tag}..master --format=%b{separator}".split(" "), check=True, stdout=subprocess.PIPE, ) - bodies = completed_process.stdout.decode("UTF-8").split("\n") + bodies = completed_process.stdout.decode("UTF-8").split(separator) assert len(titles) == len(bodies) @@ -71,22 +74,73 @@ def read_git_commit_history_since_tag(tag) -> Tuple[List[Tuple[str, str]], List[ else: print(f"No type found, skipping commit '{title}'..") + print("feats", feats) + print("fixes", fixes) + print("hotfixes", hotfixes) + return feats, fixes, hotfixes -def format_commits(*commits: List[Tuple[str, str]]) -> List[str]: +def format_commits(commits: List[Tuple[str, str]]) -> List[str]: """Format the given commits for writing to the Changelog. - The expected input str format is: + The expected input Tuple[str, str] format is: - [(FEAT|FIX|HOTFIX)-#123] - - + ([(FEAT|FIX|HOTFIX)-#123] , ) The output format is as follows:: - - #123 - + r'- #123 \n \n' + Newlines in the body are honored, and each line indented by 4 spaces automatically. + TODO: Duplicate Issues should share a single Changelog Entry. """ + if not commits: + return [] + pattern = re.compile(COMMIT_PATTERN) + formatted = set() + print("UNPACKING", commits) + for title, body in commits: + match = pattern.match(title) + issue, subject = match.groupdict()["ISSUE"], match.groupdict()["SUBJECT"] + + entry = f"- {issue} {subject}\n" + + if body: + # Make sure the body is indented by 8 spaces. + formatted_body = " ".join(body.split("\n")) + entry += f"{formatted_body}\n" + formatted.add(entry) + return sorted(formatted) + + +def update_chlog(tag: str, feats: List[str], fixes: List[str], hotfixes: List[str], chlog_path: pathlib.Path = pathlib.Path("CHANGELOG.rst")): + try: + history = chlog_path.read_text() + except FileNotFoundError: + print("No Changelog file found - creating a new one.") + history = "" + + chlog_entry = f"RELEASE {tag}\n=============\n\n" + + if feats: + feats = "\n".join(feats) + chlog_entry += f"Features\n--------\n{feats}\n""" + + if fixes: + fixes = "\n".join(fixes) + + chlog_entry += f"Fixes\n-----\n{fixes}\n" + + if hotfixes: + hotfixes = "\n".join(hotfixes) + + chlog_entry += f"Hotfixes\n--------\n{hotfixes}\n" + chlog_path.write_text(f"{chlog_entry}\n{history}") + +if __name__ == '__main__': + import sys + chlog_path = pathlib.Path(sys.argv[1]) + feats, fixes, hotfixes = read_git_commit_history_since_tag("v0.3.0") + update_chlog("0.4.0", format_commits(feats), format_commits(fixes), format_commits(hotfixes), chlog_path) \ No newline at end of file From eac2fa04aa943c918ff584063e8fd39abae66f57 Mon Sep 17 00:00:00 2001 From: brainbot-devops Date: Thu, 22 Aug 2019 10:46:34 +0200 Subject: [PATCH 3/4] Generate chlog in bumpversion step. --- .circleci/config.yml | 5 ----- .circleci/scripts/bump-part.py | 7 +++++++ .../{update-changelog.py => chlogger.py} | 17 +++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) rename .circleci/scripts/{update-changelog.py => chlogger.py} (91%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ff112488..1bacee3e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -233,11 +233,6 @@ jobs: command: | python3 ${CI_SCRIPTS_DIR}/bump-part.py - - run: - name: Update Changelog entries - command: | - python3 ${CI_SCRIPTS_DIR}/update-changelog.py {PROJECT_ROOT}/CHANGELOG.rst - # Publish a wheel and tarball of the Scenario Player to pypi. deploy-to-pypi: executor: default-executor diff --git a/.circleci/scripts/bump-part.py b/.circleci/scripts/bump-part.py index a08dd7920..abc9b30db 100755 --- a/.circleci/scripts/bump-part.py +++ b/.circleci/scripts/bump-part.py @@ -12,6 +12,7 @@ REPO_NAME, REPO_OWNER, ) +from chlogger import make_chlog from scenario_player import __version__ @@ -65,6 +66,7 @@ def get_last_tag(): part = "minor" break + print(f"Bumping part {part}..") @@ -85,6 +87,11 @@ def get_last_tag(): stdout=subprocess.PIPE, ) +if CURRENT_BRANCH == "master" and COMMIT_TYPE == "RELEASE": + r = subprocess.run(f"git --git-dir={PROJECT_GIT_DIR} describe --abbrev=0 --tags", check=True) + tag = r.stdout.decode("UTF-8").strip(" ").strip("\n") + make_chlog(Path(f"{PROJECT_ROOT}/CHANGES.rst"), new_version=tag) + print("Push Bump commit..") subprocess.run( f"git --git-dir={PROJECT_GIT_DIR} push --set-upstream origin {CURRENT_BRANCH}".split(" "), diff --git a/.circleci/scripts/update-changelog.py b/.circleci/scripts/chlogger.py similarity index 91% rename from .circleci/scripts/update-changelog.py rename to .circleci/scripts/chlogger.py index 399684650..3cd326609 100644 --- a/.circleci/scripts/update-changelog.py +++ b/.circleci/scripts/chlogger.py @@ -4,11 +4,7 @@ import subprocess from typing import List, Set, Tuple -from constants import PROJECT_GIT_DIR, CURRENT_BRANCH, COMMIT_PATTERN - - -if CURRENT_BRANCH != "master": - exit(0) +from constants import PROJECT_GIT_DIR, CURRENT_BRANCH, COMMIT_PATTERN, COMMIT_TYPE def latest_tag(): @@ -139,8 +135,9 @@ def update_chlog(tag: str, feats: List[str], fixes: List[str], hotfixes: List[st chlog_path.write_text(f"{chlog_entry}\n{history}") -if __name__ == '__main__': - import sys - chlog_path = pathlib.Path(sys.argv[1]) - feats, fixes, hotfixes = read_git_commit_history_since_tag("v0.3.0") - update_chlog("0.4.0", format_commits(feats), format_commits(fixes), format_commits(hotfixes), chlog_path) \ No newline at end of file +def make_chlog(chlog_path, new_version): + feats, fixes, hotfixes = read_git_commit_history_since_tag(new_version) + update_chlog("0.4.0", format_commits(feats), format_commits(fixes), format_commits(hotfixes), chlog_path) + + subprocess.run(f"git --git-dir={PROJECT_GIT_DIR} git add CHANGELOG.rst".split(" "), check=True) + subprocess.run(f"git --git-dir={PROJECT_GIT_DIR} git commit CHANGELOG.rst -m \"Update Changelog.\"".split(" "), check=True) From 8de75cc91b875957cbcfc4f18d1122cfcb5b1249 Mon Sep 17 00:00:00 2001 From: brainbot-devops Date: Thu, 22 Aug 2019 11:00:40 +0200 Subject: [PATCH 4/4] Remove unused code. --- .circleci/scripts/chlogger.py | 42 +++++++++++++++-------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/.circleci/scripts/chlogger.py b/.circleci/scripts/chlogger.py index 3cd326609..44a0cfd1a 100644 --- a/.circleci/scripts/chlogger.py +++ b/.circleci/scripts/chlogger.py @@ -1,26 +1,14 @@ -"""TODO: This script is a stub.""" import pathlib import re import subprocess -from typing import List, Set, Tuple +from typing import List, Tuple from constants import PROJECT_GIT_DIR, CURRENT_BRANCH, COMMIT_PATTERN, COMMIT_TYPE -def latest_tag(): - proc =subprocess.run( - f"git --git-dir={PROJECT_GIT_DIR} tag -l".split(" "), - check=True, - stdout=subprocess.PIPE, - ) - - all_tags = proc.stdout.decode("UTF-8").split("\n") - releases = sorted([tag for tag in all_tags if 'dev' not in tag and tag.startswith("v")], reverse=True) - latest, *_ = releases - return latest - - -def read_git_commit_history_since_tag(tag) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]], List[Tuple[str, str]]]: +def read_git_commit_history_since_tag( + tag + ) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]], List[Tuple[str, str]]]: """Return a list of all git commit titles since the given `tag`. If `tag` is not given, we'll use the previous tag and compare the log up @@ -70,10 +58,6 @@ def read_git_commit_history_since_tag(tag) -> Tuple[List[Tuple[str, str]], List[ else: print(f"No type found, skipping commit '{title}'..") - print("feats", feats) - print("fixes", fixes) - print("hotfixes", hotfixes) - return feats, fixes, hotfixes @@ -95,7 +79,6 @@ def format_commits(commits: List[Tuple[str, str]]) -> List[str]: return [] pattern = re.compile(COMMIT_PATTERN) formatted = set() - print("UNPACKING", commits) for title, body in commits: match = pattern.match(title) issue, subject = match.groupdict()["ISSUE"], match.groupdict()["SUBJECT"] @@ -110,7 +93,13 @@ def format_commits(commits: List[Tuple[str, str]]) -> List[str]: return sorted(formatted) -def update_chlog(tag: str, feats: List[str], fixes: List[str], hotfixes: List[str], chlog_path: pathlib.Path = pathlib.Path("CHANGELOG.rst")): +def update_chlog( + tag: str, + feats: List[str], + fixes: List[str], + hotfixes: List[str], + chlog_path: pathlib.Path = pathlib.Path("CHANGELOG.rst"), +): try: history = chlog_path.read_text() except FileNotFoundError: @@ -137,7 +126,12 @@ def update_chlog(tag: str, feats: List[str], fixes: List[str], hotfixes: List[st def make_chlog(chlog_path, new_version): feats, fixes, hotfixes = read_git_commit_history_since_tag(new_version) - update_chlog("0.4.0", format_commits(feats), format_commits(fixes), format_commits(hotfixes), chlog_path) + update_chlog( + "0.4.0", format_commits(feats), format_commits(fixes), format_commits(hotfixes), chlog_path + ) subprocess.run(f"git --git-dir={PROJECT_GIT_DIR} git add CHANGELOG.rst".split(" "), check=True) - subprocess.run(f"git --git-dir={PROJECT_GIT_DIR} git commit CHANGELOG.rst -m \"Update Changelog.\"".split(" "), check=True) + subprocess.run( + f"git --git-dir={PROJECT_GIT_DIR} git commit CHANGELOG.rst -m \"Update Changelog.\"".split(" "), + check=True + )