From f9fd90af5ad8233ccab54baf8a61a35e9134d311 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Thu, 17 Dec 2020 11:45:11 +0000 Subject: [PATCH 01/12] Update check_release for GitHub Actions * Update to use GHA environment variables * Remove checks which are handled by GHA configuration -- we will configure CI to only run check_release on PRs to master Signed-off-by: Joshua Lock --- check_release.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/check_release.py b/check_release.py index e4067f6..7d2e586 100644 --- a/check_release.py +++ b/check_release.py @@ -15,11 +15,11 @@ Check that specification updates are performed according to the versioning requirements in README.rst. - Expects Travis environment variables: - - TRAVIS_BRANCH - - TRAVIS_PULL_REQUEST_BRANCH - (see https://docs.travis-ci.com/user/environment-variables/) - + Expects GitHub Actions environment variables: + - GITHUB_REF the ref that triggered the workflow (i.e refs/pull/33/merge) + - GITHUB_BASE_REF the target branch (usually master) + - GITHUB_HEAD_REF the name of the submitters branch + (see https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables) """ import os import re @@ -40,7 +40,7 @@ class SpecError(Exception): """Common error message part. """ def __init__(self, msg): super().__init__( - msg + " please see 'Versioning' section in README.rst for details.") + msg + ", please see 'Versioning' section in README.rst for details.") def get_spec_head(): @@ -87,26 +87,11 @@ def main(): the last modified date and version number in the specification document header are higher than in the master branch, i.e. were bumped. """ - # Skip version and date comparison on push builds ... - # As per https://docs.travis-ci.com/user/environment-variables/ - # if the current job is a push build, this [env] variable is empty ("") - if not os.environ.get("TRAVIS_PULL_REQUEST_BRANCH"): - print("skipping version and date check for non pr builds ...") - sys.exit(0) - - # ... also skip on PRs that don't target the master branch - # As per https://docs.travis-ci.com/user/environment-variables/: - # for builds triggered by a pull request this [env variable] is the name of - # the branch targeted by the pull request - if not os.environ.get("TRAVIS_BRANCH") == "master": - print("skipping version and date for builds that don't target master ...") - sys.exit(0) - # Check that the current branch is based off of the master branch try: subprocess.run( - shlex.split("git merge-base --is-ancestor master {}".format( - os.environ["TRAVIS_PULL_REQUEST_BRANCH"])), check=True) + shlex.split("git merge-base --is-ancestor origin/master {}".format( + os.environ["GITHUB_REF"].lstrip("refs/"))), check=True) except subprocess.CalledProcessError as e: raise SpecError("make sure the current branch is based off of master") @@ -136,7 +121,7 @@ def main(): # Assert version bump type depending on the PR originating branch # - if the originating branch is 'draft', it must be a major (x)or minor bump # - otherwise, it must be a patch bump - if os.environ["TRAVIS_PULL_REQUEST_BRANCH"] == "draft": + if os.environ["GITHUB_BASE_REF"] == "draft": if not (((version_new[0] > version_prev[0]) != (version_new[1] > version_prev[1])) and (version_new[2] == version_prev[2])): @@ -160,4 +145,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() From 56bcbc2ac715326559b1c05c67d252176e9cd0aa Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Tue, 13 Oct 2020 08:09:31 -0700 Subject: [PATCH 02/12] GHA: run check_release on PR Run our check_release script on each PR to the specification Signed-off-by: Joshua Lock --- .github/workflows/pr.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..b2a671a --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,18 @@ +name: CI +on: + pull_request: {} + +jobs: + main: + name: Build, Validate and Deploy + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # we want all refs for the --is-ancestor check + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Check date and version + run: python check_release.py From d508ee4d03b897774317906ed37e93d5211da587 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Wed, 27 Jan 2021 12:24:17 +0000 Subject: [PATCH 03/12] Remove Travis CI configuration Signed-off-by: Joshua Lock --- .travis.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1037e43..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python - -# Disable auto-cloning and ... -git: - clone: false - -# ... instead manually fetch and checkout the pull request source branch, which -# is expected by check_release.py -# https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally -install: - - git clone --depth=50 https://github.com/${TRAVIS_REPO_SLUG}.git ${TRAVIS_REPO_SLUG} - - cd ${TRAVIS_REPO_SLUG} - - git fetch origin pull/${TRAVIS_PULL_REQUEST}/head:${TRAVIS_PULL_REQUEST_BRANCH} - - git checkout -qf ${TRAVIS_PULL_REQUEST_BRANCH} - -script: - - python check_release.py From 8280e43d9524e134d2a6c1d12e52c790fb8df331 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Mon, 15 Feb 2021 21:10:48 +0000 Subject: [PATCH 04/12] Add simple scripting to build specification releases Add some make driven automation to build releases of the specification and maintain an HTML index of releases. To be driven by a GitHub actions workflow calling make release with, i.e. mkdir build && cd build && make -f ../Makefile release Adds some helper scripts: * get_version.py: is a simple Python script which uses a regex to fetch the version number from a tuf-spec.md -- the script helps avoid dealing with different options between GNU grep (Linux) and BSD grep (macOS and other BSDs) * build_index.py: includes a string which replicates enough of the style of the bikeshed formatted specification to build an appropriately themed HTML page linking to latest, draft, and versioned releases Signed-off-by: Joshua Lock --- Makefile | 20 ++++- build_index.py | 203 +++++++++++++++++++++++++++++++++++++++++++++++++ get_version.py | 44 +++++++++++ 3 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 build_index.py create mode 100644 get_version.py diff --git a/Makefile b/Makefile index 9d56cc7..4c0ca38 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,19 @@ SHELL=/bin/bash -o pipefail -.PHONY: local +SPEC_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +.PHONY: spec -local: tuf-spec.md - bikeshed spec tuf-spec.md tuf-spec.html +spec: $(SPEC_DIR)/tuf-spec.md + bikeshed spec $(SPEC_DIR)/tuf-spec.md tuf-spec.html + +latest: spec + mkdir -p latest + cp tuf-spec.html latest/index.html + +versioned: spec + mkdir -p $(shell python3 $(SPEC_DIR)/get_version.py $(SPEC_DIR)/tuf-spec.md) + cp tuf-spec.html $(shell python3 $(SPEC_DIR)/get_version.py $(SPEC_DIR)/tuf-spec.md)/index.html + +index: + python3 $(SPEC_DIR)/build_index.py + +release: spec latest versioned diff --git a/build_index.py b/build_index.py new file mode 100644 index 0000000..c937de9 --- /dev/null +++ b/build_index.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +""" + + build_index.py + + + Joshua Lock + + + Feb 1, 2021 + + + See LICENSE-MIT for licensing information. + + + Quick and dirty script to generate an index of published specification + versions. + + Style cribbed from the bikeshed W3C theme we are using in our bikeshed + generated specification documents. +""" + +import os +import sys + +from subprocess import run + +html_header = """ + + + + The Update Framework Specification + + + + +
+

The Update Framework Specification

+
+
+
    +""" + +html_footer = """
+ + +""" + +def sanity_check(): + branch = None + + try: + branch = run("git branch --show-current".split(), capture_output=True).stdout + except Exception: + pass + + if branch != b"gh-pages\n": + print(f"build_index.py must be run from the 'gh-pages' branch (on '{branch}'") + sys.exit() + +def build_index(): + # sanity_check() + + html = html_header + + html_locations = ['latest', 'draft'] + dir_contents = sorted(os.listdir('.'), reverse=True) + for path in dir_contents: + if path.startswith('v'): + if not os.path.exists(f'{path}/index.html'): + continue + html_locations.append(path) + + for loc in html_locations: + link = f"
  • {loc}
    • \n" + html = html + link + + html = html + html_footer + + return html + +if __name__ == "__main__": + html = build_index() + with open('index.html', 'w') as index: + index.write(html) diff --git a/get_version.py b/get_version.py new file mode 100644 index 0000000..c2474f0 --- /dev/null +++ b/get_version.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +""" + + get_version.py + + + Joshua Lock + + + Feb 2, 2021 + + + See LICENSE-MIT for licensing information. + + + Quick and dirty script to get the version number from tuf-spec.md + + Unfortunately GNU grep and BSD grep take different options... +""" + +import re +import sys + +pattern = re.compile("VERSION (\d+\.\d+\.\d+)") + +def get_version(): + out = '' + spec = 'tuf-spec.md' + + if (len(sys.argv) > 1): + spec = sys.argv[1] + + with open(spec, 'r') as spec: + for line in spec: + for match in re.finditer(pattern, line): + if match.group(): + break + out = match.groups()[0] + + return f'v{out}' + +if __name__ == "__main__": + print(get_version()) From 6665dd5f78ed47bc74a8e9ceb3f6a2691e6532e1 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 15 Dec 2020 13:25:30 +0000 Subject: [PATCH 05/12] Add GitHub Actions workflow to publish releases Add a workflow that will trigger on pushes to master which: * Builds the HTML version of the spec (using the Makefile) in a build directory * Moves the built HTML file as index.html to a versioned subdirectory of build * Moves the built HTML file as index.html to a 'latest' subdirectory of build * Switches to the 'gh-pages' branch and moves the conents of the build directory into the repo root * Builds an index of published versions as index.html at the root of the repo root * Commits the generated HTML files and pushes them to the gh-pages branch on GitHub Signed-off-by: Joshua Lock --- .github/workflows/release.yml | 81 +++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7868e26 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: Release specification +on: + workflow_dispatch: + push: + branches: master + +jobs: + make-release: + name: Make and publish spec release + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@3105fb18c05ddd93efea5f9e0bef7a03a6e9e7df + with: + python-version: 3.x + + - name: Find pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6 + with: + # Use the os dependent pip cache directory found above + path: ${{ steps.pip-cache.outputs.dir }} + # A match with 'key' counts as cache hit + key: ${{ runner.os }}-pip- + + - name: Clone main + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + with: + fetch-depth: 0 + + - name: Get previous version + id: prevver + run: | + prev_version=`git tag | sort -V -r | head -n 1 | cut -c 2-` + echo "::set-output name=prev_version::$(echo -n $prev_version)" + + - name: Get version + id: getver + run: | + spec_version=`grep -oP 'VERSION \K(\d+\.\d+\.\d+)' tuf-spec.md` + echo "::set-output name=spec_version::$(echo -n $spec_version)" + + - name: Make release + if: steps.getver.outputs.spec_version != steps.prevver.outputs.prev_version + uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ steps.getver.outputs.spec_version }} + release_name: v${{ steps.getver.outputs.spec_version }} + body: Specification release v${{ steps.getver.outputs.spec_version }} + + - name: Build specification + if: steps.getver.outputs.spec_version != steps.prevver.outputs.prev_version + run: | + python -m pip install bikeshed + mkdir build && cd build + make -f ../Makefile release + + - name: Switch branch + if: steps.getver.outputs.spec_version != steps.prevver.outputs.prev_version + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + with: + ref: gh-pages + clean: false + + - name: Push generated specification + if: steps.getver.outputs.spec_version != steps.prevver.outputs.prev_version + run: | + git config user.name "TUF Specification Automation" + git config user.email tuf-spec-bot@example.com + rm -fr latest + mv build/* . + rmdir build + make index + git add . + git commit -m "Publish latest specification" + git push From 5e8d14ef7ab0c46cbb6acf4fd6eb3b1674edf771 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 16 Feb 2021 10:11:48 +0000 Subject: [PATCH 06/12] Adapt simple scripting to build draft spec Add a rule to the Makefile to move draft builds into a draft subdirectory Signed-off-by: Joshua Lock --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4c0ca38..98dcd2b 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,10 @@ latest: spec mkdir -p latest cp tuf-spec.html latest/index.html +draft: spec + mkdir -p draft + cp tuf-spec.html draft/index.html + versioned: spec mkdir -p $(shell python3 $(SPEC_DIR)/get_version.py $(SPEC_DIR)/tuf-spec.md) cp tuf-spec.html $(shell python3 $(SPEC_DIR)/get_version.py $(SPEC_DIR)/tuf-spec.md)/index.html From ad64b49e1d725c22e74817e512536b2083a79540 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 16 Feb 2021 10:12:07 +0000 Subject: [PATCH 07/12] Add GitHub actions workflow for updating draft Add a workflow that will trigger on pushes to draft which: * Builds the HTML version of the spec (using the Makefile) in a build subdirectory * Moves the built HTML file as an index.html to a 'draft' subdirectory * Switches to the gh-pages branch and moves the contents of the build subdirectory into the repo root * Commits the generated HTML files and pushes them to the gh-pages branch on GitHub Signed-off-by: Joshua Lock --- .github/workflows/draft.yml | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/draft.yml diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml new file mode 100644 index 0000000..f102b38 --- /dev/null +++ b/.github/workflows/draft.yml @@ -0,0 +1,53 @@ +name: Update draft specification +on: + workflow_dispatch: + push: + branches: draft + +jobs: + build-draft: + name: Make draft spec release + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@3105fb18c05ddd93efea5f9e0bef7a03a6e9e7df + with: + python-version: 3.x + + - name: Find pip cache dir + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@26968a09c0ea4f3e233fdddbafd1166051a095f6 + with: + # Use the os dependent pip cache directory found above + path: ${{ steps.pip-cache.outputs.dir }} + # A match with 'restore-keys' is used as fallback + key: ${{ runner.os }}-pip- + + - name: Clone + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + + - name: Build specification + run: | + python -m pip install bikeshed + mkdir build && cd build + make -f ../Makefile draft + + - name: Switch branch + uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f + with: + ref: gh-pages + clean: false + + - name: Push generated specification + run: | + git config user.name "TUF Specification Automation" + git config user.email tuf-spec-bot@example.com + rm -fr draft + mv build/* . + rmdir build + git add . + git commit -m "Publish latest specification" + git push From 9e2507a7731082066042b01f4d682e70cab68832 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Mon, 22 Feb 2021 11:05:20 +0000 Subject: [PATCH 08/12] Remove sanity check from build_index.py Our GitHub Actions workflow ensures we are on the correct branch when running the script during release, and preventing the script from being used on an arbitrary branch makes testing/debugging harder. Signed-off-by: Joshua Lock --- build_index.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/build_index.py b/build_index.py index c937de9..8485e5a 100644 --- a/build_index.py +++ b/build_index.py @@ -164,21 +164,7 @@ """ -def sanity_check(): - branch = None - - try: - branch = run("git branch --show-current".split(), capture_output=True).stdout - except Exception: - pass - - if branch != b"gh-pages\n": - print(f"build_index.py must be run from the 'gh-pages' branch (on '{branch}'") - sys.exit() - def build_index(): - # sanity_check() - html = html_header html_locations = ['latest', 'draft'] From a45879c2919b3e2523f8e296805a055fc7ecd493 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Mon, 22 Feb 2021 11:07:35 +0000 Subject: [PATCH 09/12] Fixup PR workflow Make the name more indicative of the work being performed and ensure we are only running the job on pushes to the master branch. Signed-off-by: Joshua Lock --- .github/workflows/pr.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b2a671a..a6b696c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,10 +1,12 @@ name: CI on: - pull_request: {} + pull_request: + branches: + - master jobs: main: - name: Build, Validate and Deploy + name: Check date and version are updated runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 From 47869c03f8f56a29d4c443063ca93cd8f5a1eb98 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 23 Feb 2021 10:37:45 +0000 Subject: [PATCH 10/12] Clean up scripts * Remove spurious message from check_release script * Cleanup whitespace in build_index Signed-off-by: Joshua Lock --- build_index.py | 8 ++++---- check_release.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/build_index.py b/build_index.py index 8485e5a..54f136f 100644 --- a/build_index.py +++ b/build_index.py @@ -67,17 +67,17 @@ p { margin: 1em 0; } - + dd > p:first-child, li > p:first-child { margin-top: 0; } - + ul, ol { margin-left: 0; padding-left: 2em; } - + li { margin: 0.25em 0 0.5em; padding: 0; @@ -178,7 +178,7 @@ def build_index(): for loc in html_locations: link = f"
  • {loc}
    • \n" html = html + link - + html = html + html_footer return html diff --git a/check_release.py b/check_release.py index 7d2e586..e56f7c4 100644 --- a/check_release.py +++ b/check_release.py @@ -140,7 +140,6 @@ def main(): print("*"*68) print("thanks for correctly bumping version and last modified date. :)") - print("don't forget to tag the release and to sync 'draft' with master!! :P") print("*"*68) From a9b307598cdfe52b38c0fc8336ab69e705c64d12 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 23 Feb 2021 10:39:02 +0000 Subject: [PATCH 11/12] Minor improvements to draft workflow Use the mailing list address for the bot's git user.email. Clarify that this is a draft specification commit. Signed-off-by: Joshua Lock --- .github/workflows/draft.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/draft.yml b/.github/workflows/draft.yml index f102b38..e4f5ccd 100644 --- a/.github/workflows/draft.yml +++ b/.github/workflows/draft.yml @@ -44,10 +44,10 @@ jobs: - name: Push generated specification run: | git config user.name "TUF Specification Automation" - git config user.email tuf-spec-bot@example.com + git config user.email theupdateframework@googlegroups.com rm -fr draft mv build/* . rmdir build git add . - git commit -m "Publish latest specification" + git commit -m "Publish latest draft specification" git push From 948022fd8f57cf35355f00de86ea14d226c3001b Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Tue, 23 Feb 2021 10:40:34 +0000 Subject: [PATCH 12/12] Improve release workflow * Include the specification version in the commit message when adding the built version to the gh-pages branch. * Use the mailing list address for the automation's git user.email Signed-off-by: Joshua Lock --- .github/workflows/release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7868e26..ef69df7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -69,13 +69,15 @@ jobs: - name: Push generated specification if: steps.getver.outputs.spec_version != steps.prevver.outputs.prev_version + env: + SPEC_VERSION: v${{ steps.getver.outputs.spec_version }} run: | git config user.name "TUF Specification Automation" - git config user.email tuf-spec-bot@example.com + git config user.email theupdateframework@googlegroups.com rm -fr latest mv build/* . rmdir build make index git add . - git commit -m "Publish latest specification" + git commit -m "Publish latest specification $SPEC_VERSION" git push