From c5c788e72699fcd37a80f9e61a26a535564af641 Mon Sep 17 00:00:00 2001 From: Himanshu Raj Date: Fri, 17 Feb 2017 17:23:50 -0800 Subject: [PATCH 1/2] Build cleanup + release process --- Makefile | 22 +++--- RELEASE.md | 34 ++++++++ scripts/build.sh | 162 ++++++++++++++++++++++++++++++++++++++ scripts/release.sh | 188 +++++++++------------------------------------ 4 files changed, 244 insertions(+), 162 deletions(-) create mode 100644 RELEASE.md create mode 100755 scripts/build.sh diff --git a/Makefile b/Makefile index d32aa48..f290c4b 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ # this is the classic first makefile target, and it's also the default target # run when `make` is invoked with no specific target. -all: release +all: build -# release creates a release package for contiv. +# build creates a release package for contiv. # It uses a pre-built image specified by BUILD_VERSION. -release: +build: rm -rf release/ - @bash ./scripts/release.sh + @bash ./scripts/build.sh # Brings up a demo cluster to install Contiv on - by default this is a docker, centos cluster. # It can be configured to start a RHEL cluster by setting CONTIV_NODE_OS=rhel7. @@ -33,20 +33,22 @@ demo-swarm: make cluster make install-test-swarm -# Create a release and test the release installation on a vagrant cluster +# Create a build and test the release installation on a vagrant cluster +# This should only be used with devbuild, else install-test-kubeadm will fail to +# obtain the release bits from github # TODO: The vagrant part of this can be optimized by taking snapshots instead # of creating a new set of VMs for each case -release-test-kubeadm: release +release-test-kubeadm: build # Test kubeadm (centos by default) CONTIV_KUBEADM=1 make cluster CONTIV_KUBEADM=1 make install-test-kubeadm -release-test-swarm: release +release-test-swarm: build # Test swarm (centos by default) make cluster make install-test-swarm -release-test-kubelegacy: release +release-test-kubelegacy: build # Test k8s ansible (centos by default) make cluster make install-test-kube-legacy @@ -67,6 +69,6 @@ install-test-swarm: # ci does everything necessary for a Github PR-triggered CI run. # currently, this means building a container image and running # all of the available tests. -ci: release install-test-swarm install-test-kubeadm +ci: build install-test-swarm install-test-kubeadm -.PHONY: all release cluster cluster-destroy release-test-swarm release-test-kubeadm release-test-kubelegacy install-test-swarm install-test-kubeadm install-test-kube-legacy +.PHONY: all build cluster cluster-destroy release-test-swarm release-test-kubeadm release-test-kubelegacy install-test-swarm install-test-kubeadm install-test-kube-legacy diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..fe5c351 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,34 @@ +# Automated releases +TBD + +# Manual releases +1. Set BUILD_VERSION, e.g., 1.0.0-beta.3. Set or update the following variables under script/build.sh. Merge via PR to appropriate branch. + + ``` + auth_proxy_version=${CONTIV_API_PROXY_VERSION:-"1.0.0-beta.3"} + aci_gw_version=${CONTIV_ACI_GW_VERSION:-"latest"} + contiv_version=${CONTIV_VERSION:-"1.0.0-beta.3"} + etcd_version=${CONTIV_ETCD_VERSION:-2.3.7} + docker_version=${CONTIV_DOCKER_VERSION:-1.12.6} + ``` + +2. Build docker binary image. This would create a docker image contiv/install:$BUILD_VERSION. + + ``` + make build + ``` + +2. Execute ```./scripts/release.sh``` Creates a new release on GitHub. + + ``` + export GITHUB_USER=contiv + export GITHUB_TOKEN= + ./scripts/release.sh + ``` + +3. Push image to docker hub + + ``` + docker login -u $docker_user -p $docker_password + docker push contiv/install:$BUILD_VERSION + ``` diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..04dfcdf --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +set -euo pipefail + +DEV_IMAGE_NAME="devbuild" +IMAGE_NAME="contiv/auth_proxy" +VERSION=${BUILD_VERSION-$DEV_IMAGE_NAME} + +auth_proxy_version=${CONTIV_API_PROXY_VERSION:-"1.0.0-beta.3"} +aci_gw_version=${CONTIV_ACI_GW_VERSION:-"latest"} +contiv_version=${CONTIV_VERSION:-"1.0.0-beta.3"} +etcd_version=${CONTIV_ETCD_VERSION:-2.3.7} +docker_version=${CONTIV_DOCKER_VERSION:-1.12.6} + +function usage { + echo "Usage:" + echo "./release.sh -a -c -e -p " + exit 1 +} + +function error_ret { + echo "" + echo $1 + exit 1 +} + +while getopts ":a:p:c:e:" opt; do + case $opt in + a) + aci_gw_version=$OPTARG + ;; + c) + contiv_version=$OPTARG + ;; + e) + etcd_version=$OPTARG + ;; + p) + auth_proxy_version=$OPTARG + ;; + :) + echo "An argument required for $OPTARG was not passed" + usage + ;; + ?) + usage + ;; + esac +done + +release_dir="release" +output_dir="$release_dir/contiv-$VERSION/" +output_file="$release_dir/contiv-$VERSION.tgz" +tmp_output_file="contiv-$VERSION.tgz" +full_output_file="$release_dir/contiv-full-$VERSION.tgz" +tmp_full_output_file="contiv-full-$VERSION.tgz" + +# Clean older dist folders and release binaries +rm -rf $output_dir +rm -rf $output_file + +# Release files +# k8s - install.sh to take the args and construct contiv.yaml as required and to launch kubectl +# swarm - install.sh launches the container to do the actual installation +# Top level install.sh which will either take k8s/swarm install params and do the required. +mkdir -p $output_dir +cp -rf install $output_dir +cp -rf scripts/generate-certificate.sh $output_dir/install + +# Get the ansible support files +chmod +x $output_dir/install/genInventoryFile.py +chmod +x $output_dir/install/generate-certificate.sh + + +# This is maybe optional - but assume we need it for +curl -sSL https://github.com/contiv/netplugin/releases/download/$contiv_version/netplugin-$contiv_version.tar.bz2 -o $output_dir/netplugin-$contiv_version.tar.bz2 +pushd $output_dir +tar xf netplugin-$contiv_version.tar.bz2 netctl +rm -f netplugin-$contiv_version.tar.bz2 +git clone http://github.com/contiv/ansible +popd + +# Replace versions +ansible_yaml_dir=$output_dir/install/ansible/ +ansible_env=$ansible_yaml_dir/env.json + +k8s_yaml_dir=$output_dir/install/k8s/ +contiv_yaml=$k8s_yaml_dir/contiv.yaml +aci_gw_yaml=$k8s_yaml_dir/aci_gw.yaml +auth_proxy_yaml=$k8s_yaml_dir/auth_proxy.yaml +etcd_yaml=$k8s_yaml_dir/etcd.yaml + +sed -i.bak "s/__CONTIV_VERSION__/$contiv_version/g" $contiv_yaml +sed -i.bak "s/__API_PROXY_VERSION__/$auth_proxy_version/g" $auth_proxy_yaml +sed -i.bak "s/__ACI_GW_VERSION__/$aci_gw_version/g" $aci_gw_yaml +sed -i.bak "s/__ETCD_VERSION__/$etcd_version/g" $etcd_yaml + +sed -i.bak "s/__DOCKER_VERSION__/$docker_version/g" $ansible_env +sed -i.bak "s/__CONTIV_VERSION__/$contiv_version/g" $ansible_env +sed -i.bak "s/__ACI_GW_VERSION__/$aci_gw_version/g" $ansible_env +sed -i.bak "s/__API_PROXY_VERSION__/$auth_proxy_version/g" $ansible_env +sed -i.bak "s/__ETCD_VERSION__/$etcd_version/g" $ansible_env + +chmod +x $k8s_yaml_dir/install.sh +chmod +x $k8s_yaml_dir/uninstall.sh +sed -i.bak "s/__CONTIV_INSTALL_VERSION__/$VERSION/g" $ansible_yaml_dir/install_swarm.sh +sed -i.bak "s/__CONTIV_INSTALL_VERSION__/$VERSION/g" $ansible_yaml_dir/uninstall_swarm.sh +chmod +x $ansible_yaml_dir/install_swarm.sh +chmod +x $ansible_yaml_dir/uninstall_swarm.sh +chmod +x $output_dir/install/ansible/install.sh +chmod +x $output_dir/install/ansible/uninstall.sh +# Cleanup the backup files +rm -f $k8s_yaml_dir/*.bak +rm -f $ansible_yaml_dir/*.bak + +# Build the docker container for ansible installation +ansible_spec=$output_dir/install/ansible/Dockerfile +docker build -t contiv/install:$VERSION -f $ansible_spec $output_dir + +rm -rf $output_dir/scripts +echo "**************************************************************************************************" +echo " Please ensure that contiv/install:$VERSION is pushed to docker hub" +echo "**************************************************************************************************" + +# Clean up the Dockerfiles, they are not part of the release bits. +rm -f $ansible_spec + +# Create the binary cache folder +binary_cache=$output_dir/contiv_cache +mkdir -p $binary_cache + +# Create the minimal tar bundle +tar czf $tmp_output_file -C $release_dir . + +# Save the auth proxy & aci-gw images for packaging the full docker images with contiv install binaries +if [ "$(docker images -q contiv/auth_proxy:$auth_proxy_version 2>/dev/null)" == "" ]; then + docker pull contiv/auth_proxy:$auth_proxy_version +fi +proxy_image=$(docker images -q contiv/auth_proxy:$auth_proxy_version) +docker save $proxy_image -o $binary_cache/auth-proxy-image.tar + +if [ "$(docker images -q contiv/aci-gw:$aci_gw_version 2>/dev/null)" == "" ]; then + docker pull contiv/aci-gw:$aci_gw_version +fi +aci_image=$(docker images -q contiv/aci-gw:$aci_gw_version) +docker save $aci_image -o $binary_cache/aci-gw-image.tar + +curl -sL -o $binary_cache/netplugin-$contiv_version.tar.bz2 https://github.com/contiv/netplugin/releases/download/$contiv_version/netplugin-$contiv_version.tar.bz2 + +env_file=$output_dir/install/ansible/env.json +sed -i.bak "s#.*auth_proxy_local_install.*# \"auth_proxy_local_install\": True,#g" $env_file +sed -i.bak "s#.*contiv_network_local_install.*# \"contiv_network_local_install\": True#g" $env_file + +# Create the full tar bundle +tar czf $tmp_full_output_file -C $release_dir . + + +mv $tmp_output_file $output_file +mv $tmp_full_output_file $full_output_file +rm -rf $output_dir + +echo "Success: Contiv Installer version $VERSION is available at $output_file" diff --git a/scripts/release.sh b/scripts/release.sh index 04dfcdf..0ff990a 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -1,162 +1,46 @@ #!/bin/bash - -set -euo pipefail - -DEV_IMAGE_NAME="devbuild" -IMAGE_NAME="contiv/auth_proxy" -VERSION=${BUILD_VERSION-$DEV_IMAGE_NAME} - -auth_proxy_version=${CONTIV_API_PROXY_VERSION:-"1.0.0-beta.3"} -aci_gw_version=${CONTIV_ACI_GW_VERSION:-"latest"} -contiv_version=${CONTIV_VERSION:-"1.0.0-beta.3"} -etcd_version=${CONTIV_ETCD_VERSION:-2.3.7} -docker_version=${CONTIV_DOCKER_VERSION:-1.12.6} - -function usage { - echo "Usage:" - echo "./release.sh -a -c -e -p " - exit 1 -} - -function error_ret { - echo "" - echo $1 - exit 1 -} - -while getopts ":a:p:c:e:" opt; do - case $opt in - a) - aci_gw_version=$OPTARG - ;; - c) - contiv_version=$OPTARG - ;; - e) - etcd_version=$OPTARG - ;; - p) - auth_proxy_version=$OPTARG - ;; - :) - echo "An argument required for $OPTARG was not passed" - usage - ;; - ?) - usage - ;; - esac -done - -release_dir="release" -output_dir="$release_dir/contiv-$VERSION/" -output_file="$release_dir/contiv-$VERSION.tgz" -tmp_output_file="contiv-$VERSION.tgz" -full_output_file="$release_dir/contiv-full-$VERSION.tgz" -tmp_full_output_file="contiv-full-$VERSION.tgz" - -# Clean older dist folders and release binaries -rm -rf $output_dir -rm -rf $output_file - -# Release files -# k8s - install.sh to take the args and construct contiv.yaml as required and to launch kubectl -# swarm - install.sh launches the container to do the actual installation -# Top level install.sh which will either take k8s/swarm install params and do the required. -mkdir -p $output_dir -cp -rf install $output_dir -cp -rf scripts/generate-certificate.sh $output_dir/install - -# Get the ansible support files -chmod +x $output_dir/install/genInventoryFile.py -chmod +x $output_dir/install/generate-certificate.sh - - -# This is maybe optional - but assume we need it for -curl -sSL https://github.com/contiv/netplugin/releases/download/$contiv_version/netplugin-$contiv_version.tar.bz2 -o $output_dir/netplugin-$contiv_version.tar.bz2 -pushd $output_dir -tar xf netplugin-$contiv_version.tar.bz2 netctl -rm -f netplugin-$contiv_version.tar.bz2 -git clone http://github.com/contiv/ansible -popd - -# Replace versions -ansible_yaml_dir=$output_dir/install/ansible/ -ansible_env=$ansible_yaml_dir/env.json - -k8s_yaml_dir=$output_dir/install/k8s/ -contiv_yaml=$k8s_yaml_dir/contiv.yaml -aci_gw_yaml=$k8s_yaml_dir/aci_gw.yaml -auth_proxy_yaml=$k8s_yaml_dir/auth_proxy.yaml -etcd_yaml=$k8s_yaml_dir/etcd.yaml - -sed -i.bak "s/__CONTIV_VERSION__/$contiv_version/g" $contiv_yaml -sed -i.bak "s/__API_PROXY_VERSION__/$auth_proxy_version/g" $auth_proxy_yaml -sed -i.bak "s/__ACI_GW_VERSION__/$aci_gw_version/g" $aci_gw_yaml -sed -i.bak "s/__ETCD_VERSION__/$etcd_version/g" $etcd_yaml - -sed -i.bak "s/__DOCKER_VERSION__/$docker_version/g" $ansible_env -sed -i.bak "s/__CONTIV_VERSION__/$contiv_version/g" $ansible_env -sed -i.bak "s/__ACI_GW_VERSION__/$aci_gw_version/g" $ansible_env -sed -i.bak "s/__API_PROXY_VERSION__/$auth_proxy_version/g" $ansible_env -sed -i.bak "s/__ETCD_VERSION__/$etcd_version/g" $ansible_env - -chmod +x $k8s_yaml_dir/install.sh -chmod +x $k8s_yaml_dir/uninstall.sh -sed -i.bak "s/__CONTIV_INSTALL_VERSION__/$VERSION/g" $ansible_yaml_dir/install_swarm.sh -sed -i.bak "s/__CONTIV_INSTALL_VERSION__/$VERSION/g" $ansible_yaml_dir/uninstall_swarm.sh -chmod +x $ansible_yaml_dir/install_swarm.sh -chmod +x $ansible_yaml_dir/uninstall_swarm.sh -chmod +x $output_dir/install/ansible/install.sh -chmod +x $output_dir/install/ansible/uninstall.sh -# Cleanup the backup files -rm -f $k8s_yaml_dir/*.bak -rm -f $ansible_yaml_dir/*.bak - -# Build the docker container for ansible installation -ansible_spec=$output_dir/install/ansible/Dockerfile -docker build -t contiv/install:$VERSION -f $ansible_spec $output_dir - -rm -rf $output_dir/scripts -echo "**************************************************************************************************" -echo " Please ensure that contiv/install:$VERSION is pushed to docker hub" -echo "**************************************************************************************************" - -# Clean up the Dockerfiles, they are not part of the release bits. -rm -f $ansible_spec - -# Create the binary cache folder -binary_cache=$output_dir/contiv_cache -mkdir -p $binary_cache - -# Create the minimal tar bundle -tar czf $tmp_output_file -C $release_dir . - -# Save the auth proxy & aci-gw images for packaging the full docker images with contiv install binaries -if [ "$(docker images -q contiv/auth_proxy:$auth_proxy_version 2>/dev/null)" == "" ]; then - docker pull contiv/auth_proxy:$auth_proxy_version +# Assumes following variables to be defined: +# OLD_VERSION - previous version against which to create changelog +# BUILD_VERSION - new version being released +# GITHUB_USER - contiv +# GITHUB_TOKEN - your github token +if [ -z "$OLD_VERSION" ]; then + echo "A release requires OLD_VERSION to be defined" + exit 1 fi -proxy_image=$(docker images -q contiv/auth_proxy:$auth_proxy_version) -docker save $proxy_image -o $binary_cache/auth-proxy-image.tar -if [ "$(docker images -q contiv/aci-gw:$aci_gw_version 2>/dev/null)" == "" ]; then - docker pull contiv/aci-gw:$aci_gw_version +if [ "$OLD_VERSION" != "none" ]; then + comparison="$OLD_VERSION..HEAD" fi -aci_image=$(docker images -q contiv/aci-gw:$aci_gw_version) -docker save $aci_image -o $binary_cache/aci-gw-image.tar +pre_release="-p" -curl -sL -o $binary_cache/netplugin-$contiv_version.tar.bz2 https://github.com/contiv/netplugin/releases/download/$contiv_version/netplugin-$contiv_version.tar.bz2 +if [ "$OLD_VERSION" != "none" ]; then + changelog=$(git log $comparison --oneline --no-merges --reverse) -env_file=$output_dir/install/ansible/env.json -sed -i.bak "s#.*auth_proxy_local_install.*# \"auth_proxy_local_install\": True,#g" $env_file -sed -i.bak "s#.*contiv_network_local_install.*# \"contiv_network_local_install\": True#g" $env_file + if [ -z "$changelog" ]; then + echo "No new changes to release!" + exit 0 + fi +else + changelog="don't forget to update the changelog" +fi + +# Install github-release binary if not present +[ -n "`which github-release`" ] || go get -u github.com/aktau/github-release || exit 1 -# Create the full tar bundle -tar czf $tmp_full_output_file -C $release_dir . +TAR_FILENAME="contiv-"${BUILD_VERSION}".tgz" +TAR_FILENAME2="contiv-full-"${BUILD_VERSION}".tgz" +TAR_FILE="../release/contiv-"${BUILD_VERSION}".tgz" +TAR_FILE2="../release/contiv-full-"${BUILD_VERSION}".tgz" +if [ ! -f ${TAR_FILE} ] || [ ! -f ${TAR_FILE2} ]; then + echo "release file(s) does not exist" + exit 1 +fi -mv $tmp_output_file $output_file -mv $tmp_full_output_file $full_output_file -rm -rf $output_dir +set -x +( ( github-release -v release $pre_release -r install -t $BUILD_VERSION -d "**Changelog**
$changelog" ) && \ + ( ( github-release -v upload -r install -t $BUILD_VERSION -n $TAR_FILENAME -f $TAR_FILE && \ + github-release -v upload -r install -t $BUILD_VERSION -n $TAR_FILENAME2 -f $TAR_FILE2 ) || \ + github-release -v delete -r install -t $BUILD_VERSION ) ) || exit 1 -echo "Success: Contiv Installer version $VERSION is available at $output_file" From fceedcd5000f83ddb5c62e12e704099623cfaff2 Mon Sep 17 00:00:00 2001 From: Himanshu Raj Date: Tue, 21 Feb 2017 12:18:16 -0800 Subject: [PATCH 2/2] Comment fixes + Clarifications based on feedback --- Makefile | 8 ++++---- RELEASE.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f290c4b..0d65175 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,8 @@ cluster-destroy: # demo-k8s brings up a cluster with k8s, runs the installer on it, and shows the URL # of the demo Contiv Admin Console which was set up # BUILD_VERSION must be setup to use a specific build, e.g. -# export BUILD_VERSION-1.0.0-beta.3 +# export BUILD_VERSION=1.0.0-beta.3 +# Or run make as BUILD_VERSION=1.0.0-beta.3 make demo-k8s demo-k8s: CONTIV_KUBEADM=1 make cluster CONTIV_KUBEADM=1 make install-test-kubeadm @@ -28,14 +29,13 @@ demo-k8s: # demo-swarm brings up a cluster with docker swarm, runs the installer on it, and shows the URL # of the demo Contiv Admin Console which was set up # BUILD_VERSION must be setup to use a specific build, e.g. -# export BUILD_VERSION-1.0.0-beta.3 +# export BUILD_VERSION=1.0.0-beta.3 +# Or run make as BUILD_VERSION=1.0.0-beta.3 make demo-k8s demo-swarm: make cluster make install-test-swarm # Create a build and test the release installation on a vagrant cluster -# This should only be used with devbuild, else install-test-kubeadm will fail to -# obtain the release bits from github # TODO: The vagrant part of this can be optimized by taking snapshots instead # of creating a new set of VMs for each case release-test-kubeadm: build diff --git a/RELEASE.md b/RELEASE.md index fe5c351..382a85c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ TBD docker_version=${CONTIV_DOCKER_VERSION:-1.12.6} ``` -2. Build docker binary image. This would create a docker image contiv/install:$BUILD_VERSION. +2. Build docker binary image. This would create a docker image contiv/install:$BUILD_VERSION. It also creates two release bundles - contiv-${BUILD_VETSION}.tgz and contiv-full-${BUILD_VERSION}.tgz. This version should be tested locally using a vagrant setup with release-test-* make targets. ``` make build