From 1453439f33833114d32646c57e183ade464dea4d Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Mon, 23 Apr 2018 11:57:44 -0400 Subject: [PATCH 1/5] remove unnecessary paragraph --- jobs/CHANGELOG.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/jobs/CHANGELOG.rst b/jobs/CHANGELOG.rst index a1f450e8..8be0a022 100644 --- a/jobs/CHANGELOG.rst +++ b/jobs/CHANGELOG.rst @@ -67,9 +67,6 @@ Changelog - pin all requirements, fixes #353 (#356) - -This document describes changes between each past release. - 1.1.5 (2018-02-22) ------------------ From f73a4dd19d999b4860138732a915895cd8461072 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Mon, 23 Apr 2018 14:40:08 -0400 Subject: [PATCH 2/5] make-release script --- README.md | 51 ++++---- bin/make-release.py | 302 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 331 insertions(+), 22 deletions(-) create mode 100755 bin/make-release.py diff --git a/README.md b/README.md index bbdf8218..52b595b7 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ _Buildhub_ aims to provide a public database of comprehensive information about ## Development -1. Install Docker -2. To run tests: `make test` -3. To lint check Python code: `make lintcheck` +1. Install Docker +2. To run tests: `make test` +3. To lint check Python code: `make lintcheck` ## Continuous Integration @@ -24,22 +24,29 @@ for all continous integration. ## Releasing -We don't use `zest.releaser` right now because of some problems with -releasing a package that is not at the root of the repo (`jobs/`), and -because we have no interest in uploading this project to PyPI, but -this could change if we figure out how. - -The current procedure is: - -* Bump version in `jobs/setup.py` -* Update the release date in `jobs/CHANGELOG.rst` -* `git commit -am "Bump x.y.z"` -* Open PR, wait for it to become green -* Merge PR -* `git tag x.y.z` -* `git push --tags origin` -* `make lambda.zip` -* Add a release on Github with the lambda.zip attached -* [Click here][bugzilla-link] to open a ticket to get it deployed - -[bugzilla-link]: https://bugzilla.mozilla.org/enter_bug.cgi?comment=Could%20you%20please%20update%20the%20lambda%20function%20for%20Buildhub%20with%20the%20following%20one%3F%0D%0A%0D%0A%5BInsert%20a%20short%20description%20of%20the%20changes%20here.%5D%0D%0A%0D%0Ahttps%3A%2F%2Fgithub.com%2Fmozilla-services%2Fbuildhub%2Freleases%2Ftag%2FX.Y.Z%0D%0A%0D%0Ahttps%3A%2F%2Fgithub.com%2Fmozilla-services%2Fbuildhub%2Freleases%2Fdownload%2FX.Y.Z%2Fbuildhub-lambda-X.Y.Z.zip%0D%0A%0D%0AThanks%21&component=Operations%3A%20Storage&product=Cloud%20Services&qa_contact=chartjes%40mozilla.com&short_desc=Please%20deploy%20buildhub%20lambda%20function%20X.Y.Z +To make a release you need to have write access to +`github.com/mozilla-services/buildhub`. First you have to generate a +new `lambda.zip` file by running: + + make lambda.zip + +(This is generated inside Docker). + +Then you need a [GitHub Personal Access Token](https://github.com/settings/tokens) +with `repos` scope. This is to generate GitHub Releases and upload assets +to them. Next, run `./bin/make-release.py`. The only required parameter +is the "type" of the release. The choices are: + +* `major` (e.g. '2.6.9' to '3.0.0') +* `minor` (e.g. '2.6.7' to '2.7.0') +* `patch` (e.g. '2.6.7' to '2.6.8') + +Like this for example: + + GITHUB_API_KEY=895f...ce09 ./bin/make-release.py minor + +This will bump the version in `setup.py`, update the `CHANGELOG.rst` and +make a tag and push that tag to GitHub. + +Then, it will create a Release and upload the latest `lambda.zip` as an +attachment to that Release. diff --git a/bin/make-release.py b/bin/make-release.py new file mode 100755 index 00000000..a36e9f36 --- /dev/null +++ b/bin/make-release.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import datetime +import re +import subprocess +import time +import pkg_resources + +import requests + +OWNER = 'mozilla-services' +REPO = 'buildhub' + + +def _format_age(seconds): + seconds = int(seconds) + if seconds > 3600: + return '{} hours {} minutes ago'.format( + seconds // 3600, + round(seconds % 3600 / 60), + ) + elif seconds > 60: + return '{} minutes {} seconds ago'.format( + seconds // 60, + round(seconds % 60), + ) + else: + return '{} seconds ago'.format(seconds) + + +def _format_file_size(bytes): + if bytes > 1024 * 1024: + return '{:.1f}MB'.format(bytes / 1024 / 1024) + elif bytes > 1024: + return '{:.1f}KB'.format(bytes / 1024) + else: + return '{}B'.format(bytes) + + +def main( + part, + dry_run=False, + github_api_key=None, + tag_name_format='v{version}', + upstream_name='master', +): + github_api_key = github_api_key or os.environ['GITHUB_API_KEY'] + assert github_api_key, 'GITHUB_API_KEY or --github-api-key not set.' + + # If this 401 errors, go here to generate a new personal access token: + # https://github.com/settings/tokens + # Give it all the 'repos' scope. + api_url = 'https://api.github.com/user' + response = requests.get(api_url, headers={ + 'Authorization': 'token {}'.format(github_api_key) + }) + response.raise_for_status() + + # Before we proceed, check that the `lambda.zip` is up to date. + lambda_mtime = os.stat('lambda.zip').st_mtime + age = time.time() - lambda_mtime + print("The lambda.zip...\n") + dt = datetime.datetime.fromtimestamp(lambda_mtime) + print('\tLast modified:', dt.strftime('%d %B %Y - %H:%M:%S')) + print('\tAge:', _format_age(age)) + print('') + try: + ok = input('Is this lambda.zip recently generated? [Y/n] ') + except KeyboardInterrupt: + print('\n\nTip! Generate it by running: make lambda.zip ') + return + if ok.lower().strip() == 'n': + print('Tip! Generate it by running: make lambda.zip ') + return + + # Figure out the current version + current_version = pkg_resources.get_distribution('buildhub').version + z, y, x = [int(x) for x in current_version.split('.')] + if part == 'major': + next_version = (z + 1, 0, 0) + elif part == 'minor': + next_version = (z, y + 1, 0) + else: + next_version = (z, y, x + 1) + next_version = '.'.join(str(n) for n in next_version) + + # Figure out the CHANGELOG + + # Let's make sure we're up-to-date + current_branch = subprocess.check_output( + 'git rev-parse --abbrev-ref HEAD'.split() + ).decode('utf-8').strip() + if current_branch != 'master': + # print("WARNING, NOT ON MASTER BRANCH")# DELETE WHEN DONE HACKING + print("Must be on the master branch to do this") + return 1 + + # The current branch can't be dirty + try: + subprocess.check_call( + 'git diff --quiet --ignore-submodules HEAD'.split() + ) + except subprocess.CalledProcessError: + print( + "Can't be \"git dirty\" when we're about to git pull. " + "Stash or commit what you're working on." + ) + return 2 + + # Make sure we have all the old git tags + subprocess.check_output( + f'git pull {upstream_name} master --tags'.split(), + stderr=subprocess.STDOUT + ) + + # We're going to use the last tag to help you write a tag message + last_tag, last_tag_message = subprocess.check_output([ + 'git', + 'for-each-ref', + '--sort=-taggerdate', + '--count=1', + '--format', + '%(tag)|%(contents:subject)', + 'refs/tags' + ]).decode('utf-8').strip().split('|', 1) + + commits_since = subprocess.check_output( + f'git log {last_tag}..HEAD --oneline'.split() + ).decode('utf-8') + commit_messages = [] + for commit in commits_since.splitlines(): + wo_sha = re.sub('^[a-f0-9]{7} ', '', commit) + commit_messages.append(wo_sha) + + print(' NEW CHANGE LOG '.center(80, '=')) + change_log = [] + head = '{} ({})'.format( + next_version, + datetime.datetime.now().strftime('%Y-%m-%d') + ) + head += '\n{}'.format('-' * len(head)) + change_log.append(head) + change_log.extend(['- {}'.format(x) for x in commit_messages]) + print('\n\n'.join(change_log)) + print('=' * 80) + + assert commit_messages + + # Edit jobs/setup.py + with open('jobs/setup.py') as f: + setup_py = f.read() + assert "version='{}',".format(current_version) in setup_py + setup_py = setup_py.replace( + "version='{}',".format(current_version), + "version='{}',".format(next_version), + ) + if not dry_run: + with open('jobs/setup.py', 'w') as f: + f.write(setup_py) + + # Edit jobs/CHANGELOG.rst + with open('jobs/CHANGELOG.rst') as f: + original = f.read() + assert '\n\n'.join(change_log) not in original + new_change_log = original.replace( + '=========', + '=========\n\n{}\n\n'.format( + '\n\n'.join(change_log) + ) + ) + if not dry_run: + with open('jobs/CHANGELOG.rst', 'w') as f: + f.write(new_change_log) + + # Actually commit this change. + commit_message = f'Bump {next_version}' + if dry_run: + print('git add jobs/CHANGELOG.rst jobs/setup.py') + print( + f'git commit -m "{commit_message}"' + ) + else: + subprocess.check_call([ + 'git', 'add', 'jobs/CHANGELOG.rst', 'jobs/setup.py', + ]) + subprocess.check_call([ + 'git', 'commit', '-m', commit_message, + ]) + + # Commit these changes + tag_name = tag_name_format.format(version=next_version) + tag_body = '\n\n'.join(['- {}'.format(x) for x in commit_messages]) + if dry_run: + print( + f'git tag -s -a {tag_name} -m "...See CHANGELOG output above..."' + ) + else: + subprocess.check_call([ + 'git', + 'tag', + '-s', + '-a', tag_name, + '-m', tag_body, + ]) + + # Let's push this now + if dry_run: + print(f'git push {upstream_name} master --tags') + else: + subprocess.check_call( + f'git push {upstream_name} master --tags'.split() + ) + + if not dry_run: + release = _create_release( + github_api_key, + tag_name, + tag_body, + name=tag_name, + ) + asset_info = _upload_lambda_zip( + github_api_key, + release['upload_url'], + release['id'], + f'buildhub-lambda-{tag_name}.zip', + ) + print('Build asset uploaded.') + print('Can be downloaded at:') + print(asset_info['browser_download_url']) + + print('\n') + print(' 🎉 ALL DONE! 🎊 ') + print('\n') + + +def _create_release(github_api_key, tag_name, body, name=''): + api_url = ( + f'https://api.github.com' + f'/repos/{OWNER}/{REPO}/releases' + ) + response = requests.post( + api_url, + json={ + 'tag_name': tag_name, + 'body': body, + 'name': name, + }, headers={ + 'Authorization': 'token {}'.format(github_api_key) + } + ) + response.raise_for_status() + return response.json() + + +def _upload_lambda_zip(github_api_key, upload_url, release_id, filename): + upload_url = upload_url.replace( + '{?name,label}', + f'?name={filename}', + ) + print('Uploading lambda.zip as {} ({})...'.format( + filename, + _format_file_size(os.stat('lambda.zip').st_size) + )) + with open('lambda.zip', 'rb') as f: + response = requests.get( + upload_url, + data=f, + headers={ + 'Content-Type': 'application/zip', + 'Authorization': 'token {}'.format(github_api_key) + }, + ) + response.raise_for_status() + return response.json() + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description='Unleash Buildhub!') + parser.add_argument('part', type=str, help='major, minor or patch') + parser.add_argument('-d', '--dry-run', action='store_true') + parser.add_argument( + '-g', '--github-api-key', + help='GitHub API key unless set by GITHUB_API_KEY env var.' + ) + parser.add_argument( + '-u', '--upstream-name', + help=( + 'Name of the git remote origin to push to. Not your fork. ' + 'Defaults to "origin".' + ), + default='origin', + ) + args = parser.parse_args() + if args.part not in ('major', 'minor', 'patch'): + raise ValueError('invalid part. Must be major, minor, or patch') + args = vars(args) + main(args.pop('part'), **args) From 7d4cdbbf96be191ed0b117bf1bf0515aac29e32b Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 24 Apr 2018 07:57:37 -0400 Subject: [PATCH 3/5] deployment-bug.py --- README.md | 18 ++++++ bin/deployment-bug.py | 131 ++++++++++++++++++++++++++++++++++++++++++ bin/make-release.py | 4 +- 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100755 bin/deployment-bug.py diff --git a/README.md b/README.md index 52b595b7..8709356e 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,21 @@ make a tag and push that tag to GitHub. Then, it will create a Release and upload the latest `lambda.zip` as an attachment to that Release. + +## Deployment + +The outlined steps above only upgrade the cron job part of Buildhub. +And only for Stage is it automatically upgraded simply by making a new +Release. + +At the time of writing we still need to file a Bugzilla bug to have +the Lambda job upgraded on Stage. [Issue #423](https://github.com/mozilla-services/buildhub/issues/423) +is about automating this away. + +To upgrade the Lambda job on **Stage** run: + + ./bin/deployment-bug.py stage-lambda + +To upgrade the cron job _and_ Lambda job on **Prod** run: + + ./bin/deployment-bug.py prod diff --git a/bin/deployment-bug.py b/bin/deployment-bug.py new file mode 100755 index 00000000..8eff538e --- /dev/null +++ b/bin/deployment-bug.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +from urllib.parse import urlencode + +import requests + + +OWNER = 'mozilla-services' +REPO = 'buildhub' + +VALID_ENVIRONMENTS = ('stage', 'prod') +VALID_TASKS = ('cron', 'lambda', 'both') + +QA_CONTACT = 'chartjes@mozilla.com' + + +def main( + environment, + task, + tag=None, + dry_run=False, +): + api_url = f'https://api.github.com/repos/{OWNER}/{REPO}/releases' + if not tag: + api_url += '/latest' + else: + api_url += f'/tags/{tag}' + response = requests.get(api_url) + response.raise_for_status() + release_info = response.json() + + # Prepare some variables for the string templates + release_url = release_info['html_url'] + lambda_asset_url = None + for asset in release_info['assets']: + lambda_asset_url = asset['browser_download_url'] + env = environment.title() + release_tag_name = release_info['tag_name'] + + if task == 'lambda': + summary = f"On {env}, please deploy Buildhub Lambda function {release_tag_name}" # noqa + comment = f""" + Could you please update the Lambda function for Buildhub {env} with the following one? + + {release_url} + + {lambda_asset_url} + + Thanks! + """ # noqa + elif task == 'cron': + summary = f"On {env}, please deploy Buildhub Cron function {release_tag_name}" # noqa + comment = f""" + Could you please update the Cron function for Buildhub {env} with the following one? + + {release_url} + + Thanks! + """ # noqa + else: + summary = f"On {env}, please deploy Buildhub Cron and Lambda function {release_tag_name}" # noqa + comment = f""" + Could you please update the Cron *and* Lambda function for Buildhub {env} with the following one? + + {release_url} + + {lambda_asset_url} + + Thanks! + """ # noqa + + comment = '\n'.join(x.strip() for x in comment.strip().splitlines()) + params = { + 'qa_contact': QA_CONTACT, + 'comment': comment, + 'short_desc': summary, + 'component': 'Operations: Storage', + 'product': 'Cloud Services', + 'bug_file_loc': release_url, + } + URL = 'https://bugzilla.mozilla.org/enter_bug.cgi?' + urlencode(params) + print('To file this bug, click (or copy) this URL:') + print('👇') + print(URL) + print('👆') + return 0 + + +if __name__ == '__main__': + import argparse + + def check_environment(value): + value = value.strip() + if value not in VALID_ENVIRONMENTS: + raise argparse.ArgumentTypeError( + f'{value!r} not in {VALID_ENVIRONMENTS}' + ) + return value + + def check_task(value): + value = value.strip() + if value not in VALID_TASKS: + raise argparse.ArgumentTypeError( + f'{value!r} not in {VALID_TASKS}' + ) + return value + + parser = argparse.ArgumentParser( + description='Deploy Buildhub (by filing Bugzilla bugs)!' + ) + parser.add_argument( + '-t', '--tag', type=str, + help=( + f'Name of the release (e.g. "v1.2.0"). If ommitted will be looked ' + f'on GitHub at https://github.com/{OWNER}/{REPO}/releases' + ) + ) + + parser.add_argument( + 'environment', help='stage or prod', type=check_environment + ) + parser.add_argument( + 'task', help='cron or lambda or both', type=check_task, + ) + parser.add_argument('-d', '--dry-run', action='store_true') + args = parser.parse_args() + args = vars(args) + main(args.pop('environment'), args.pop('task'), **args) diff --git a/bin/make-release.py b/bin/make-release.py index a36e9f36..6efecd88 100755 --- a/bin/make-release.py +++ b/bin/make-release.py @@ -280,7 +280,7 @@ def _upload_lambda_zip(github_api_key, upload_url, release_id, filename): if __name__ == '__main__': import argparse - parser = argparse.ArgumentParser(description='Unleash Buildhub!') + parser = argparse.ArgumentParser(description='Release Buildhub!') parser.add_argument('part', type=str, help='major, minor or patch') parser.add_argument('-d', '--dry-run', action='store_true') parser.add_argument( @@ -297,6 +297,6 @@ def _upload_lambda_zip(github_api_key, upload_url, release_id, filename): ) args = parser.parse_args() if args.part not in ('major', 'minor', 'patch'): - raise ValueError('invalid part. Must be major, minor, or patch') + parser.error("invalid part. Must be 'major', 'minor', or 'patch'") args = vars(args) main(args.pop('part'), **args) From fdf03b0aef5e8e1a896edc28d5fd45494be1290f Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 24 Apr 2018 12:04:22 -0400 Subject: [PATCH 4/5] utiloity function check_output --- bin/make-release.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/make-release.py b/bin/make-release.py index 6efecd88..36db0eda 100755 --- a/bin/make-release.py +++ b/bin/make-release.py @@ -41,6 +41,12 @@ def _format_file_size(bytes): return '{}B'.format(bytes) +def check_output(*args, **kwargs): + if len(args) == 1 and isinstance(args[0], str): + args = args[0].split() + return subprocess.check_output(*args, **kwargs).decode('utf-8').strip() + + def main( part, dry_run=False, @@ -91,9 +97,7 @@ def main( # Figure out the CHANGELOG # Let's make sure we're up-to-date - current_branch = subprocess.check_output( - 'git rev-parse --abbrev-ref HEAD'.split() - ).decode('utf-8').strip() + current_branch = check_output('git rev-parse --abbrev-ref HEAD') if current_branch != 'master': # print("WARNING, NOT ON MASTER BRANCH")# DELETE WHEN DONE HACKING print("Must be on the master branch to do this") @@ -112,13 +116,13 @@ def main( return 2 # Make sure we have all the old git tags - subprocess.check_output( + check_output( f'git pull {upstream_name} master --tags'.split(), stderr=subprocess.STDOUT ) # We're going to use the last tag to help you write a tag message - last_tag, last_tag_message = subprocess.check_output([ + last_tag, last_tag_message = check_output([ 'git', 'for-each-ref', '--sort=-taggerdate', @@ -126,11 +130,9 @@ def main( '--format', '%(tag)|%(contents:subject)', 'refs/tags' - ]).decode('utf-8').strip().split('|', 1) + ]).split('|', 1) - commits_since = subprocess.check_output( - f'git log {last_tag}..HEAD --oneline'.split() - ).decode('utf-8') + commits_since = check_output(f'git log {last_tag}..HEAD --oneline') commit_messages = [] for commit in commits_since.splitlines(): wo_sha = re.sub('^[a-f0-9]{7} ', '', commit) From 0639ca70a8b893a4f8f89bf5629b981e379cc1d8 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 24 Apr 2018 12:27:38 -0400 Subject: [PATCH 5/5] review fixes --- bin/make-release.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/make-release.py b/bin/make-release.py index 36db0eda..c2800820 100755 --- a/bin/make-release.py +++ b/bin/make-release.py @@ -78,10 +78,10 @@ def main( ok = input('Is this lambda.zip recently generated? [Y/n] ') except KeyboardInterrupt: print('\n\nTip! Generate it by running: make lambda.zip ') - return + return 3 if ok.lower().strip() == 'n': print('Tip! Generate it by running: make lambda.zip ') - return + return 3 # Figure out the current version current_version = pkg_resources.get_distribution('buildhub').version @@ -117,7 +117,7 @@ def main( # Make sure we have all the old git tags check_output( - f'git pull {upstream_name} master --tags'.split(), + f'git pull {upstream_name} master --tags', stderr=subprocess.STDOUT ) @@ -238,6 +238,8 @@ def main( print(' 🎉 ALL DONE! 🎊 ') print('\n') + return 0 + def _create_release(github_api_key, tag_name, body, name=''): api_url = ( @@ -281,6 +283,7 @@ def _upload_lambda_zip(github_api_key, upload_url, release_id, filename): if __name__ == '__main__': + import sys import argparse parser = argparse.ArgumentParser(description='Release Buildhub!') parser.add_argument('part', type=str, help='major, minor or patch') @@ -301,4 +304,4 @@ def _upload_lambda_zip(github_api_key, upload_url, release_id, filename): if args.part not in ('major', 'minor', 'patch'): parser.error("invalid part. Must be 'major', 'minor', or 'patch'") args = vars(args) - main(args.pop('part'), **args) + sys.exit(main(args.pop('part'), **args))