From 63945e76192abc3a4a3e09dbebc6eadbbc4250f7 Mon Sep 17 00:00:00 2001 From: James Nelson Date: Fri, 19 Jan 2024 18:51:34 -0600 Subject: [PATCH] ci: Add cocogitto/release.sh to perform releases. (#206) Fixes #213 Fixes #218 * Add cog.toml to configure cocogitto, which generates changelogs and version tags * Add tools/release.sh to perform a complete version bump + changelog update and github release for a plugin * Updated README.md with instructions regarding the release process --------- Co-authored-by: Mike Bender --- .gitignore | 5 +- README.md | 31 ++++++++ cog.toml | 79 ++++++++++++++++++++ tools/extract_changelog.sh | 27 +++++++ tools/release.sh | 123 ++++++++++++++++++++++++++++++ tools/update_version.sh | 149 +++++++++++++++++++++++++++++++++++++ tools/validate.sh | 62 +++++++++++++++ 7 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 cog.toml create mode 100755 tools/extract_changelog.sh create mode 100755 tools/release.sh create mode 100755 tools/update_version.sh create mode 100755 tools/validate.sh diff --git a/.gitignore b/.gitignore index 8e6cf67c8..a6b15029c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ tsconfig.tsbuildinfo junit.xml # Allow for local overrides of docker-compose.yml. https://docs.docker.com/compose/multiple-compose-files/merge/ -docker-compose.override.yml \ No newline at end of file +docker-compose.override.yml + +# Ignore temporary files created during a release +releases/ diff --git a/README.md b/README.md index e79d67f9c..093fb0c0e 100644 --- a/README.md +++ b/README.md @@ -188,3 +188,34 @@ services: # Specifying a data volume here will override the default data folder, and you will not be able to access the default data files (such as the demo data) - /path/to/mydata/:/data ``` + +## Release Management + +In order to manage changelogs, version bumps and github releases, we use [cocogitto](https://github.com/cocogitto/cocogitto), or `cog` for short. Follow the [Installation instructions](https://github.com/cocogitto/cocogitto?tab=readme-ov-file#installation) to install `cog`. For Linux and Windows, we recommend using [cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) to install. For MacOS, we recommend using [brew](https://brew.sh/). + +The main configuration file is cog.toml, which we run using some helper scripts located in the `tools/` directory. + +You will also need the [GitHub CLI](https://cli.github.com/) tool installed to create and push releases to GitHub. + +### Cutting a New Release + +In order to release a given plugin, you will run the script: `tools/release.sh `. +This must be done on a branch named `main` and will publish to the `git remote -v` named `origin` (you can do test releases on your fork). + +`tools/release.sh ` will validate that your system has the necessary software installed and setup correctly, then invoke `cog bump --auto --package `, +which will invoke the necessary programs and scripts to automate a version bump and GitHub release. + +During development, it is expected that all commit message will adhere to [conventional commits]([https://www.conventionalcommits.org/en/about/]). +`cog` will then uses your commit messages to compute a new version number, assemble a changelog, update our version in source code, create and push git tags, and perform a GitHub release for the given plugin. + +See `cog.toml` to understand the full details of the release process. + +After you have successfully run `tools/release.sh` once, you should be able to directly invoke `cog bump --auto --package `, or omit the `--package` to release all plugins which have updated files. + +### Updating Versions in Source Code + +As part of the release process, `cog` will, per our `cog.toml` configuration, invoke `tools/update_version.sh `, which is a script that uses `sed` to update a plugin's version number in whatever source file we happen to use as the source of truth for version information in the given plugin. + +*[WARNING]* If you change where the source of truth for a plugin's version is located, you must update `tools/update_version.sh` to update the correct file with a new version number. + +We use `tools/update_version.sh` to remove any `.dev0` "developer version" suffix before creating a release, and to put the `.dev0` version suffix back after completing the release. diff --git a/cog.toml b/cog.toml new file mode 100644 index 000000000..d84f61c28 --- /dev/null +++ b/cog.toml @@ -0,0 +1,79 @@ +from_latest_tag = true +ignore_merge_commits = false +disable_changelog = false +# we never tag any code outside the plugins/ directory. Everything else is build glue. +generate_mono_repository_global_tag = false +# limit which branches to perform bumps from +branch_whitelist = [ "main" ] +# we don't really use [skip ci] action filtering, but leaving here in case we decide to someday +skip_ci = "[skip ci]" +skip_untracked = false +tag_prefix = "v" + +# bump hooks for global versions only; we don't use global version, but leaving here for posterity +pre_bump_hooks = [] +post_bump_hooks = [] + +# bump hooks for package versions, which is what we actually use +pre_package_bump_hooks = [ + "echo Updating {{package}} to version {{version}}", + # make sure user has correct software installed and is authenticated with `gh` cli tool + "../../tools/validate.sh", + # change the version number of the given plugin in source, and commit it as a chore(version): commit + "../../tools/update_version.sh {{package}} {{version}} -d", +] +# between pre_ and post_ bump hooks, cog itself will update the checked in CHANGELOG.md file, per updated plugin. +post_package_bump_hooks = [ + # prepare the github release changelog file + "mkdir -p ../../releases", + "../../tools/extract_changelog.sh CHANGELOG.md > ../../releases/GITHUB_CHANGELOG-{{package}}.md", + # update the version number to have a .dev0 suffix (when possible, only done for python plugins) + "../../tools/update_version.sh {{package}} {{version}} --dev", + "git commit -m 'chore(version): update {{package}} version to {{version}}'", + # push the tag and the commits to main + "git push origin {{package}}-v{{version}}", + "git push origin main", + # cut a github release using our conventional-commit changelog + "gh release create {{package}}-v{{version}} --notes-file ../../releases/GITHUB_CHANGELOG-{{package}}.md --title {{package}}-v{{version}} --verify-tag" +] + +[git_hooks] + +[commit_types] +# exclude chore and ci commits from changelog entries +chore = { changelog_title = "", omit_from_changelog = true } +ci = { changelog_title = "", omit_from_changelog = true } + +[changelog] +path = "CHANGELOG.md" +template = "remote" +remote = "github.com" +repository = "deephaven-plugins" +owner = "deephaven" +authors = [ + { username = "jnumainville", signature = "Joe Numainville" }, + { username = "mofojed", signature = "Mike Bender" }, + { username = "devinrsmith", signature = "Devin Smith" }, + { username = "mattrunyon", signature = "Matt Runyon" }, + { username = "vbabich", signature = "Vlad Babich" }, + { username = "dsmmcken", signature = "Don McKenzie" }, + { username = "bmingles", signature = "Brian Ingles" }, + { username = "niloc132", signature = "Colin Alworth" }, + { username = "rachelmbrubaker", signature = "Rachel Brubaker" }, + { username = "JamesXNelson", signature = "James Nelson" }, + +] + +[bump_profiles] + + +[packages] +auth-keyclock = { path = "plugins/auth-keyclock", public_api=false } +dashboard-object-viewer = { path = "plugins/dashboard-object-viewer", public_api=false } +json = { path = "plugins/json", public_api=false } +matplotlib = { path = "plugins/matplotlib", public_api=false } +plotly = { path = "plugins/plotly", public_api=false } +plotly-express = { path = "plugins/plotly-express", public_api=false } +table-example = { path = "plugins/table-example", public_api=false } +ui = { path = "plugins/ui", public_api=false } + diff --git a/tools/extract_changelog.sh b/tools/extract_changelog.sh new file mode 100755 index 000000000..328873861 --- /dev/null +++ b/tools/extract_changelog.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# This workaround is borrowed from https://github.com/cocogitto/cocogitto/issues/300 +# It extracts the top chunk of an individual plugin's changelog into the github release changelog file. + +if [ ! -f "$1" ]; then + echo "The file passed as first argument, $1, does not exist." + exit 1 +fi + +separator_count=0 +extract_data=false + +while IFS= read -r line; do + if $extract_data && [[ "$line" != "- - -" ]]; then + echo "$line" + fi + + if [[ "$line" == "- - -" ]]; then + ((separator_count++)) + if ((separator_count == 1)); then + extract_data=true + elif ((separator_count == 2)); then + break + fi + fi +done < "$1" diff --git a/tools/release.sh b/tools/release.sh new file mode 100755 index 000000000..c846d7344 --- /dev/null +++ b/tools/release.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# This script is designed to automate the version bump + github release process. +# It requires for you to have installed both the [GitHub CLI tool](https://cli.github.com/) `gh` and the [cocogitto tool](https://github.com/cocogitto/cocogitto#installation) `cog` + +tab=$'\t' +log_prefix="$(id -un)$tab - " + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") + +function log_error() { + { echo -e "\033[31m$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- [ERROR] $* \033[0m" ; } 2> /dev/null +} 2>/dev/null + +function log_info() { + { echo "$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- $*" ; } 2>/dev/null +} 2>/dev/null + +if ! which cog >/dev/null; then + { + log_error "cog command not found!" + log_error "Installation instructions are here: https://github.com/cocogitto/cocogitto?tab=readme-ov-file#installation" + log_error "mac users can install via brew; for other OSes, you should install cargo and then use cargo to install cocogitto (cog)" + log_error "Note that cog must be on your PATH. An alias will not work." + } 2>/dev/null + exit 99 +fi + +# todo: enforce git remote named origin + +all_plugins="$(cd "$ROOT_DIR/plugins" ; find . -mindepth 1 -maxdepth 1 -type d | sed 's|./||g')" + +function usage() { + log_info "Simple utility to automate version bump + release process" + log_info "This script accepts the following arguments:" + log_info "" + log_info "--help | -h $tab-> Prints this help message" + log_info "--debug | -d $tab-> Turn on xtrace debugging" + log_info " $tab-> Runs the version bump + release for a given plugin" + log_info "Valid choices are: +$all_plugins" +} 2> /dev/null + +if [ -n "$(git status --short)" ]; then + { + log_error "Detected uncommitted files via git status:" + git status --short + log_error "Releases can only be performed with a clean git status" + log_error 'You must commit/stash your changes, or `git reset --hard` to erase them' + exit 95 + } 2>/dev/null +fi + +# Collect arguments +package= +while (( $# > 0 )); do + case "$1" in + --debug | -d) + set -o xtrace ;; + --help | -h) + usage ; exit 0 ;; + *) + if [ -n "$package" ]; then + { + log_error "Illegal argument $1. Already requested release of package '$package'" + log_error "You can only release one package at a time" + } 2>/dev/null + exit 94 + fi + if grep -qE "^$1\$" <<< "$all_plugins"; then + package="$1" + else + { + log_error "Illegal argument $1. Expected one of: +$all_plugins" + } 2>/dev/null + exit 93 + fi + ;; + esac + shift +done + +# Validate arguments +if [ -z "$package" ]; then + { + log_error "Expected exactly one package name argument" + log_error "Valid choices are: +$all_plugins" + } 2>/dev/null + exit 92 +fi + +if ! grep -q "plugins/$package" "$ROOT_DIR/cog.toml"; then + { + log_error "Did not see plugins/$package in cog.toml" + log_error "Make sure to list your plugins under the [plugins] section of cog.toml" + } 2>/dev/null + exit 91 +fi + +if [ -n "$(git status --short)" ]; then + { + log_error "Detected uncommitted files via git status:" + git status --short + log_error "Releases can only be performed with a clean git status" + log_error 'You must commit/stash your changes, or `git reset --hard` to erase them' + exit 95 + } 2>/dev/null +fi + +# Perform release +{ log_info "Releasing package '$package'" ; } 2>/dev/null +( +cd "$ROOT_DIR" +cog bump --package "$package" --auto +) diff --git a/tools/update_version.sh b/tools/update_version.sh new file mode 100755 index 000000000..3892a1873 --- /dev/null +++ b/tools/update_version.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# This script is used to update the version of a given plugin in its source code. +# Because this differs for various plugins, this script is used to hide all that complexity +# behind a very simple API. `update_version.sh ` + +# You should not need to call this script directly. +# It is invoked for you when calling the release.sh script located next to this one. + + +tab=$'\t' +log_prefix="$(id -un)$tab - " + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") + +function log_error() { + { echo -e "\033[31m$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- [ERROR] $* \033[0m" ; } 2> /dev/null +} 2>/dev/null + +function log_info() { + { echo "$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- $*" ; } 2>/dev/null +} 2>/dev/null + +all_plugins="$(cd "$ROOT_DIR/plugins" ; find . -mindepth 1 -maxdepth 1 -type d | sed 's|./||g')" + +function usage() { + log_info "Simple utility to update plugin file in order to change versions." + log_info "This script accepts the following arguments:" + log_info "" + log_info "--help | -h $tab-> Prints this help message" + log_info "--debug | -d $tab-> Turn on xtrace debugging" + log_info "--dev $tab-> Append .dev0 to python plugin versions" + log_info " $tab-> The name of the plugin that we are going to set the version for" + log_info "Valid choices are: +$all_plugins" + log_info " $tab-> Specify the new version of the given plugin" + +} 2> /dev/null + +package= +version= +dev=false +while (( $# > 0 )); do + case "$1" in + --debug | -d) + set -o xtrace ;; + --dev) + dev=true ;; + --help | -h) + usage ; exit 0 ;; + *) + if [ -z "$package" ]; then + if grep -qE "^$1\$" <<< "$all_plugins"; then + package="$1" + else + { + log_error "Illegal argument $1. Expected one of: +$all_plugins" + } 2>/dev/null + exit 93 + fi + elif [ -z "$version" ]; then + version="${1/v/}" + else + { + log_error "Illegal argument $1. Already saw package '$package' and version '$version'" + log_error "This script expects two and only two non -flag arguments, and " + } 2>/dev/null + exit 94 + fi + ;; + esac + shift +done + +if [ -z "$package" ]; then + { + log_error "Expected exactly two arguments " + log_error "Valid choices for are: +$all_plugins" + } 2>/dev/null + exit 92 +fi +if [ -z "$version" ]; then + { + log_error "Did not receive a second argument of a version to set $package to" + } 2>/dev/null + exit 91 +fi + +function update_file() { + local file="$1" + local prefix="$2" + local suffix="$3" + local extra="${4:-}" + local expected="${prefix}${version}${extra}${suffix}" + if ! grep -q "$expected" "$ROOT_DIR/plugins/$file"; then + # annoyingly, sed on mac is extremely old, so we have to handle it differently. + if [[ "$(uname)" == Darwin* ]]; then + sed -e "s/${prefix}.*/${expected}/g" -i '' "$ROOT_DIR/plugins/$file" + else + sed -e "s/${prefix}.*/${expected}/g" -i "$ROOT_DIR/plugins/$file" + fi + git add "$ROOT_DIR/plugins/$file" + fi +} + +extra= +[ "$dev" = true ] && extra=".dev0" +case "$package" in + auth-keycloak) + update_file auth-keycloak/src/js/package.json '"version": "' '",' + ;; + dashboard-object-viewer) + update_file dashboard-object-viewer/src/js/package.json '"version": "' '",' + ;; + json) + update_file json/src/deephaven/plugin/json/__init__.py '__version__ = "' '"' "$extra" + ;; + matplotlib) + update_file matplotlib/setup.cfg 'version = ' '' "$extra" + ;; + plotly) + update_file plotly/src/deephaven/plugin/plotly/__init__.py '__version__ = "' '"' "$extra" + ;; + plotly-express) + update_file plotly-express/setup.cfg 'version = ' '' "$extra" + ;; + table-example) + update_file table-example/src/js/package.json '"version": "' '",' + ;; + ui) + update_file ui/src/js/package.json '"version": "' '",' + update_file ui/src/deephaven/ui/__init__.py '__version__ = "' '"' "$extra" + ;; + *) + { + log_error "Unhandled plugin $package. You will need to add wiring in $SCRIPT_NAME" + exit 90 + } +esac + +log_info "Done updating $package version to $version${extra}" diff --git a/tools/validate.sh b/tools/validate.sh new file mode 100755 index 000000000..8bd92eed6 --- /dev/null +++ b/tools/validate.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# This script is called as a cog pre-bump hook to validate that your runtime environment +# has the necessary programs installed, and github auth status to successful perform a release. + +tab=$'\t' +log_prefix="$(id -un)$tab - " + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]}") + +function log_error() { + { echo -e "\033[31m$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- [ERROR] $* \033[0m" ; } 2> /dev/null +} 2>/dev/null + +function log_info() { + { echo "$log_prefix $(date "+%Y-%m-%d %H:%M:%S")$tab |--- $*" ; } 2>/dev/null +} 2>/dev/null + +if ! which cog >/dev/null; then + { + log_error "cog command not found!" + log_error "Installation instructions are here: https://github.com/cocogitto/cocogitto?tab=readme-ov-file#installation" + log_error "mac users can install via brew; for other OSes, you should install cargo and then use cargo to install cocogitto (cog)" + log_error "Note that cog must be on your PATH. An alias will not work." + } 2>/dev/null + exit 99 +fi +if ! which gh >/dev/null; then + { + log_error "gh command not found!" + log_error "Installation instructions are here: https://github.com/cli/cli?tab=readme-ov-file#installation" + exit 98 + } 2>/dev/null +fi +if ! gh auth status; then + { + log_error "You must be logged into gh to continue!" + log_error 'Run `gh auth login`' + exit 97 + } 2>/dev/null +fi + +# we need to run a gh command to ensure you've set the gh repo already. +# we also can't pipe the output of this command to /dev/null, or else gh will always exit with code 0 +# so, we'll prepare the user for some noise. +echo +log_info "Listing previous release to ensure gh is setup correctly:" +if ! gh release list --limit 1 2>/dev/stdout; then + { + log_error "You must select the correct gh repo to continue!" + log_error 'Run `gh repo set-default git@github.com:deephaven/deephaven-plugins.git`' + exit 96 + } 2>/dev/null +fi +echo +