From 4b817d9c6de27b210a0b534becbd11353e585dde Mon Sep 17 00:00:00 2001 From: Nikhil Barthwal <46331366+nbarthwal@users.noreply.github.com> Date: Thu, 4 Apr 2019 11:01:00 -0700 Subject: [PATCH] Update dep for test-infra (#592) --- Gopkg.lock | 4 +- .../knative/test-infra/scripts/README.md | 10 +- .../knative/test-infra/scripts/e2e-tests.sh | 57 +++--- .../knative/test-infra/scripts/library.sh | 76 ++++++-- .../test-infra/scripts/presubmit-tests.sh | 11 +- .../knative/test-infra/scripts/release.sh | 172 +++++++++++++++--- 6 files changed, 267 insertions(+), 63 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 85430302..73479e9c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -347,14 +347,14 @@ [[projects]] branch = "master" - digest = "1:ae14885490829a664d0f19dc78852719e1d7221144e69eaa8c201e854028487c" + digest = "1:921c8888ac7bf7240bc5473d029201b3eb19f9592c7647c4c29f96b6c901b1bb" name = "github.com/knative/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "cfcb8c3d5b2eb146c40e5d8a828520f240b35747" + revision = "4ce16d390c55c71290f3b285bdfe37a1c5490304" [[projects]] digest = "1:56dbf15e091bf7926cb33a57cb6bdfc658fc6d3498d2f76f10a97ce7856f1fde" diff --git a/vendor/github.com/knative/test-infra/scripts/README.md b/vendor/github.com/knative/test-infra/scripts/README.md index 498fe0ad..07398fb7 100644 --- a/vendor/github.com/knative/test-infra/scripts/README.md +++ b/vendor/github.com/knative/test-infra/scripts/README.md @@ -33,6 +33,8 @@ This is a helper script to run the presubmit tests. To use it: - `DISABLE_MD_LINTING`: Disable linting markdown files, defaults to 0 (false). - `DISABLE_MD_LINK_CHECK`: Disable checking links in markdown files, defaults to 0 (false). + - `PRESUBMIT_TEST_FAIL_FAST`: Fail the presubmit test immediately if a test fails, + defaults to 0 (false). 1. [optional] Define the functions `pre_build_tests()` and/or `post_build_tests()`. These functions will be called before or after the @@ -130,9 +132,15 @@ This is a helper script for Knative E2E test scripts. To use it: 1. [optional] Write the `test_teardown()` function, which will tear down the test resources. +1. [optional] Write the `cluster_setup()` function, which will set up any resources + before the test cluster is created. + +1. [optional] Write the `cluster_teardown()` function, which will tear down any + resources after the test cluster is destroyed. + 1. [optional] Write the `dump_extra_cluster_state()` function. It will be called when a test fails, and can dump extra information about the current state - of the cluster (tipically using `kubectl`). + of the cluster (typically using `kubectl`). 1. [optional] Write the `parse_flags()` function. It will be called whenever an unrecognized flag is passed to the script, allowing you to define your own flags. diff --git a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh index 34b42313..c2328e7b 100755 --- a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh @@ -120,12 +120,40 @@ function save_metadata() { EOF } +# Delete target pools and health checks that might have leaked. +# See https://github.com/knative/serving/issues/959 for details. +# TODO(adrcunha): Remove once the leak issue is resolved. +function delete_leaked_network_resources() { + (( IS_BOSKOS )) && return + # Ensure we're using the GCP project used by kubetest + local gcloud_project="$(gcloud config get-value project)" + local http_health_checks="$(gcloud compute target-pools list \ + --project=${gcloud_project} --format='value(healthChecks)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ + grep httpHealthChecks | tr '\n' ' ')" + local target_pools="$(gcloud compute target-pools list \ + --project=${gcloud_project} --format='value(name)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ + tr '\n' ' ')" + if [[ -n "${target_pools}" ]]; then + echo "Found leaked target pools, deleting" + gcloud compute forwarding-rules delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} + gcloud compute target-pools delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} + fi + if [[ -n "${http_health_checks}" ]]; then + echo "Found leaked health checks, deleting" + gcloud compute http-health-checks delete -q --project=${gcloud_project} ${http_health_checks} + fi +} + # Create a test cluster with kubetest and call the current script again. function create_test_cluster() { # Fail fast during setup. set -o errexit set -o pipefail + if function_exists cluster_setup; then + cluster_setup || fail_test "cluster setup failed" + fi + header "Creating test cluster" echo "Cluster will have a minimum of ${E2E_MIN_CLUSTER_NODES} and a maximum of ${E2E_MAX_CLUSTER_NODES} nodes." @@ -167,44 +195,29 @@ function create_test_cluster() { (( SKIP_KNATIVE_SETUP )) && test_cmd_args+=" --skip-knative-setup" [[ -n "${GCP_PROJECT}" ]] && test_cmd_args+=" --gcp-project ${GCP_PROJECT}" [[ -n "${E2E_SCRIPT_CUSTOM_FLAGS[@]}" ]] && test_cmd_args+=" ${E2E_SCRIPT_CUSTOM_FLAGS[@]}" + local extra_flags=() + # If using boskos, save time and let it tear down the cluster + (( ! IS_BOSKOS )) && extra_flags+=(--down) # Don't fail test for kubetest, as it might incorrectly report test failure # if teardown fails (for details, see success() below) set +o errexit run_go_tool k8s.io/test-infra/kubetest \ kubetest "${CLUSTER_CREATION_ARGS[@]}" \ --up \ - --down \ --extract "${E2E_CLUSTER_VERSION}" \ --gcp-node-image "${SERVING_GKE_IMAGE}" \ --test-cmd "${E2E_SCRIPT}" \ --test-cmd-args "${test_cmd_args}" \ + ${extra_flags[@]} \ ${EXTRA_KUBETEST_FLAGS[@]} echo "Test subprocess exited with code $?" # Ignore any errors below, this is a best-effort cleanup and shouldn't affect the test result. set +o errexit - # Ensure we're using the GCP project used by kubetest - gcloud_project="$(gcloud config get-value project)" - # Delete target pools and health checks that might have leaked. - # See https://github.com/knative/serving/issues/959 for details. - # TODO(adrcunha): Remove once the leak issue is resolved. - local http_health_checks="$(gcloud compute target-pools list \ - --project=${gcloud_project} --format='value(healthChecks)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ - grep httpHealthChecks | tr '\n' ' ')" - local target_pools="$(gcloud compute target-pools list \ - --project=${gcloud_project} --format='value(name)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ - tr '\n' ' ')" - if [[ -n "${target_pools}" ]]; then - echo "Found leaked target pools, deleting" - gcloud compute forwarding-rules delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} - gcloud compute target-pools delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} - fi - if [[ -n "${http_health_checks}" ]]; then - echo "Found leaked health checks, deleting" - gcloud compute http-health-checks delete -q --project=${gcloud_project} ${http_health_checks} - fi + function_exists cluster_teardown && cluster_teardown + delete_leaked_network_resources local result="$(cat ${TEST_RESULT_FILE})" + echo "Artifacts were written to ${ARTIFACTS}" echo "Test result code is ${result}" - exit ${result} } diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh index e148f021..89a93c91 100755 --- a/vendor/github.com/knative/test-infra/scripts/library.sh +++ b/vendor/github.com/knative/test-infra/scripts/library.sh @@ -44,6 +44,11 @@ readonly IS_PROW readonly REPO_ROOT_DIR="$(git rev-parse --show-toplevel)" readonly REPO_NAME="$(basename ${REPO_ROOT_DIR})" +# Set ARTIFACTS to an empty temp dir if unset +if [[ -z "${ARTIFACTS:-}" ]]; then + export ARTIFACTS="$(mktemp -d)" +fi + # On a Prow job, redirect stderr to stdout so it's synchronously added to log (( IS_PROW )) && exec 2>&1 @@ -141,6 +146,25 @@ function wait_until_pods_running() { return 1 } +# Waits until all batch job pods are running in the given namespace. +# Parameters: $1 - namespace. +function wait_until_batch_job_complete() { + echo -n "Waiting until all batch job pods in namespace $1 run to completion." + for i in {1..150}; do # timeout after 5 minutes + local pods="$(kubectl get pods --selector=job-name --no-headers -n $1 2>/dev/null | grep -v '^[[:space:]]*$')" + # All pods must be complete + local not_complete=$(echo "${pods}" | grep -v Completed | wc -l) + if [[ ${not_complete} -eq 0 ]]; then + echo -e "\nAll pods are complete:\n${pods}" + return 0 + fi + echo -n "." + sleep 2 + done + echo -e "\n\nERROR: timeout waiting for pods to complete\n${pods}" + return 1 +} + # Waits until the given service has an external address (IP/hostname). # Parameters: $1 - namespace. # $2 - service name. @@ -173,7 +197,7 @@ function wait_until_routable() { for i in {1..150}; do # timeout after 5 minutes local val=$(curl -H "Host: $2" "http://$1" 2>/dev/null) if [[ -n "$val" ]]; then - echo "\nEndpoint is now routable" + echo -e "\nEndpoint is now routable" return 0 fi echo -n "." @@ -269,10 +293,6 @@ function report_go_test() { local args=" $@ " local go_test="go test -race -v ${args/ -v / }" # Just run regular go tests if not on Prow. - if (( ! IS_PROW )); then - ${go_test} - return - fi echo "Running tests with '${go_test}'" local report=$(mktemp) ${go_test} | tee ${report} @@ -287,6 +307,13 @@ function report_go_test() { | sed -e "s#\"github.com/knative/${REPO_NAME}/#\"#g" \ > ${xml} echo "XML report written to ${xml}" + if (( ! IS_PROW )); then + # Keep the suffix, so files are related. + local logfile=${xml/junit_/go_test_} + logfile=${logfile/.xml/.log} + cp ${report} ${logfile} + echo "Test log written to ${logfile}" + fi return ${failed} } @@ -294,8 +321,9 @@ function report_go_test() { function start_latest_knative_serving() { header "Starting Knative Serving" subheader "Installing Istio" - echo "Installing Istio CRD from ${KNATIVE_ISTIO_CRD_YAML}" + echo "Running Istio CRD from ${KNATIVE_ISTIO_CRD_YAML}" kubectl apply -f ${KNATIVE_ISTIO_CRD_YAML} || return 1 + wait_until_batch_job_complete istio-system || return 1 echo "Installing Istio from ${KNATIVE_ISTIO_YAML}" kubectl apply -f ${KNATIVE_ISTIO_YAML} || return 1 wait_until_pods_running istio-system || return 1 @@ -306,15 +334,6 @@ function start_latest_knative_serving() { wait_until_pods_running knative-serving || return 1 } -# Install the latest stable Knative/build in the current cluster. -function start_latest_knative_build() { - header "Starting Knative Build" - subheader "Installing Knative Build" - echo "Installing Build from ${KNATIVE_BUILD_RELEASE}" - kubectl apply -f ${KNATIVE_BUILD_RELEASE} || return 1 - wait_until_pods_running knative-build || return 1 -} - # Run a go tool, installing it first if necessary. # Parameters: $1 - tool package/dir for go get/install. # $2 - tool to run. @@ -400,6 +419,33 @@ function is_protected_gcr() { [[ -n $1 && "$1" =~ "^gcr.io/knative-(releases|nightly)/?$" ]] } +# Remove symlinks in a path that are broken or lead outside the repo. +# Parameters: $1 - path name, e.g. vendor +function remove_broken_symlinks() { + for link in $(find $1 -type l); do + # Remove broken symlinks + if [[ ! -e ${link} ]]; then + unlink ${link} + continue + fi + # Get canonical path to target, remove if outside the repo + local target="$(ls -l ${link})" + target="${target##* -> }" + [[ ${target} == /* ]] || target="./${target}" + target="$(cd `dirname ${link}` && cd ${target%/*} && echo $PWD/${target##*/})" + if [[ ${target} != *github.com/knative/* ]]; then + unlink ${link} + continue + fi + done +} + +# Return whether the given parameter is knative-tests. +# Parameters: $1 - project name +function is_protected_project() { + [[ -n "$1" && "$1" == "knative-tests" ]] +} + # Returns the canonical path of a filesystem object. # Parameters: $1 - path to return in canonical form # $2 - base dir for relative links; optional, defaults to current diff --git a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh index e3c024f8..b10a8515 100755 --- a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh @@ -22,6 +22,7 @@ source $(dirname ${BASH_SOURCE})/library.sh # Custom configuration of presubmit tests readonly DISABLE_MD_LINTING=${DISABLE_MD_LINTING:-0} readonly DISABLE_MD_LINK_CHECK=${DISABLE_MD_LINK_CHECK:-0} +readonly PRESUBMIT_TEST_FAIL_FAST=${PRESUBMIT_TEST_FAIL_FAST:-0} # Extensions or file patterns that don't require presubmit tests. readonly NO_PRESUBMIT_FILES=(\.png \.gitignore \.gitattributes ^OWNERS ^OWNERS_ALIASES ^AUTHORS) @@ -317,8 +318,14 @@ function main() { fi run_build_tests || failed=1 - run_unit_tests || failed=1 - run_integration_tests || failed=1 + # If PRESUBMIT_TEST_FAIL_FAST is set to true, don't run unit tests if build tests failed + if (( ! PRESUBMIT_TEST_FAIL_FAST )) || (( ! failed )); then + run_unit_tests || failed=1 + fi + # If PRESUBMIT_TEST_FAIL_FAST is set to true, don't run integration tests if build/unit tests failed + if (( ! PRESUBMIT_TEST_FAIL_FAST )) || (( ! failed )); then + run_integration_tests || failed=1 + fi exit ${failed} } diff --git a/vendor/github.com/knative/test-infra/scripts/release.sh b/vendor/github.com/knative/test-infra/scripts/release.sh index 139bfd52..5704576a 100755 --- a/vendor/github.com/knative/test-infra/scripts/release.sh +++ b/vendor/github.com/knative/test-infra/scripts/release.sh @@ -22,6 +22,10 @@ source $(dirname ${BASH_SOURCE})/library.sh # GitHub upstream. readonly KNATIVE_UPSTREAM="https://github.com/knative/${REPO_NAME}" +# GCRs for Knative releases. +readonly NIGHTLY_GCR="gcr.io/knative-nightly/github.com/knative/${REPO_NAME}" +readonly RELEASE_GCR="gcr.io/knative-releases/github.com/knative/${REPO_NAME}" + # Georeplicate images to {us,eu,asia}.gcr.io readonly GEO_REPLICATION=(us eu asia) @@ -58,17 +62,21 @@ function publish_yamls() { local DEST="gs://${RELEASE_GCS_BUCKET}/$1/" shift echo "Publishing [$@] to ${DEST}" - gsutil cp $@ ${DEST} + gsutil -m cp $@ ${DEST} } - # Before publishing the YAML files, cleanup the `latest` dir. - echo "Cleaning up the `latest` directory first" - gsutil -m rm gs://${RELEASE_GCS_BUCKET}/latest/** + # Before publishing the YAML files, cleanup the `latest` dir if it exists. + local latest_dir="gs://${RELEASE_GCS_BUCKET}/latest" + if [[ -n "$(gsutil ls ${latest_dir} 2> /dev/null)" ]]; then + echo "Cleaning up '${latest_dir}' first" + gsutil -m rm ${latest_dir}/** + fi verbose_gsutil_cp latest $@ [[ -n ${TAG} ]] && verbose_gsutil_cp previous/${TAG} $@ } # These are global environment variables. SKIP_TESTS=0 +PRESUBMIT_TEST_FAIL_FAST=1 TAG_RELEASE=0 PUBLISH_RELEASE=0 PUBLISH_TO_GITHUB=0 @@ -80,6 +88,8 @@ RELEASE_GCS_BUCKET="" KO_FLAGS="" VALIDATION_TESTS="./test/presubmit-tests.sh" YAMLS_TO_PUBLISH="" +FROM_NIGHTLY_RELEASE="" +FROM_NIGHTLY_RELEASE_GCS="" export KO_DOCKER_REPO="" export GITHUB_TOKEN="" @@ -89,6 +99,14 @@ function hub_tool() { run_go_tool github.com/github/hub hub $@ } +# Shortcut to "git push" that handles authentication. +# Parameters: $1..$n - arguments to "git push ". +function git_push() { + local repo_url="${KNATIVE_UPSTREAM}" + [[ -n "${GITHUB_TOKEN}}" ]] && repo_url="${repo_url/:\/\//:\/\/${GITHUB_TOKEN}@}" + git push ${repo_url} $@ +} + # Return the master version of a release. # For example, "v0.2.1" returns "0.2" # Parameters: $1 - release version label. @@ -106,6 +124,13 @@ function release_build_number() { echo "${tokens[2]}" } +# Return the short commit SHA from a release tag. +# For example, "v20010101-deadbeef" returns "deadbeef". +function hash_from_tag() { + local tokens=(${1//-/ }) + echo "${tokens[1]}" +} + # Setup the repository upstream, if not set. function setup_upstream() { # hub and checkout need the upstream URL to be set @@ -210,6 +235,86 @@ function prepare_dot_release() { fi } +# Setup source nightly image for a release. +function prepare_from_nightly_release() { + echo "Release from nightly requested" + SKIP_TESTS=1 + if [[ "${FROM_NIGHTLY_RELEASE}" == "latest" ]]; then + echo "Finding the latest nightly release" + find_latest_nightly "${NIGHTLY_GCR}" || abort "cannot find the latest nightly release" + echo "Latest nightly is ${FROM_NIGHTLY_RELEASE}" + fi + readonly FROM_NIGHTLY_RELEASE_GCS="gs://knative-nightly/${REPO_NAME}/previous/${FROM_NIGHTLY_RELEASE}" + gsutil ls -d "${FROM_NIGHTLY_RELEASE_GCS}" > /dev/null \ + || abort "nightly release ${FROM_NIGHTLY_RELEASE} doesn't exist" +} + +# Build a release from an existing nightly one. +function build_from_nightly_release() { + banner "Building the release" + echo "Fetching manifests from nightly" + local yamls_dir="$(mktemp -d)" + gsutil -m cp -r "${FROM_NIGHTLY_RELEASE_GCS}/*" "${yamls_dir}" || abort "error fetching manifests" + # Update references to release GCR + for yaml in ${yamls_dir}/*.yaml; do + sed -i -e "s#${NIGHTLY_GCR}#${RELEASE_GCR}#" "${yaml}" + done + YAMLS_TO_PUBLISH="$(find ${yamls_dir} -name '*.yaml' -printf '%p ')" + echo "Copying nightly images" + copy_nightly_images_to_release_gcr "${NIGHTLY_GCR}" "${FROM_NIGHTLY_RELEASE}" + # Create a release branch from the nightly release tag. + local commit="$(hash_from_tag ${FROM_NIGHTLY_RELEASE})" + echo "Creating release branch ${RELEASE_BRANCH} at commit ${commit}" + git checkout -b ${RELEASE_BRANCH} ${commit} || abort "cannot create branch" + git_push upstream ${RELEASE_BRANCH} || abort "cannot push branch" +} + +# Build a release from source. +function build_from_source() { + run_validation_tests ${VALIDATION_TESTS} + banner "Building the release" + build_release + # Do not use `||` above or any error will be swallowed. + if [[ $? -ne 0 ]]; then + abort "error building the release" + fi +} + +# Copy tagged images from the nightly GCR to the release GCR, tagging them 'latest'. +# This is a recursive function, first call must pass $NIGHTLY_GCR as first parameter. +# Parameters: $1 - GCR to recurse into. +# $2 - tag to be used to select images to copy. +function copy_nightly_images_to_release_gcr() { + for entry in $(gcloud --format="value(name)" container images list --repository="$1"); do + copy_nightly_images_to_release_gcr "${entry}" "$2" + # Copy each image with the given nightly tag + for x in $(gcloud --format="value(tags)" container images list-tags "${entry}" --filter="tags=$2" --limit=1); do + local path="${entry/${NIGHTLY_GCR}}" # Image "path" (remove GCR part) + local dst="${RELEASE_GCR}${path}:latest" + gcloud container images add-tag "${entry}:$2" "${dst}" || abort "error copying image" + done + done +} + +# Recurse into GCR and find the nightly tag of the first `latest` image found. +# Parameters: $1 - GCR to recurse into. +function find_latest_nightly() { + for entry in $(gcloud --format="value(name)" container images list --repository="$1"); do + find_latest_nightly "${entry}" && return 0 + for tag in $(gcloud --format="value(tags)" container images list-tags "${entry}" \ + --filter="tags=latest" --limit=1); do + local tags=( ${tag//,/ } ) + # Skip if more than one nightly tag, as we don't know what's the latest. + if [[ ${#tags[@]} -eq 2 ]]; then + local nightly_tag="${tags[@]/latest}" # Remove 'latest' tag + FROM_NIGHTLY_RELEASE="${nightly_tag// /}" # Remove spaces + return 0 + fi + done + done + return 1 +} + # Parses flags and sets environment variables accordingly. function parse_flags() { TAG="" @@ -220,6 +325,7 @@ function parse_flags() { KO_DOCKER_REPO="gcr.io/knative-nightly" RELEASE_GCS_BUCKET="knative-nightly/${REPO_NAME}" GITHUB_TOKEN="" + FROM_NIGHTLY_RELEASE="" local has_gcr_flag=0 local has_gcs_flag=0 local is_dot_release=0 @@ -236,6 +342,7 @@ function parse_flags() { --nopublish) PUBLISH_RELEASE=0 ;; --dot-release) is_dot_release=1 ;; --auto-release) is_auto_release=1 ;; + --from-latest-nightly) FROM_NIGHTLY_RELEASE=latest ;; *) [[ $# -ge 2 ]] || abort "missing parameter after $1" shift @@ -266,6 +373,10 @@ function parse_flags() { [[ ! -f "$1" ]] && abort "file $1 doesn't exist" RELEASE_NOTES=$1 ;; + --from-nightly) + [[ $1 =~ ^v[0-9]+-[0-9a-f]+$ ]] || abort "nightly tag must be 'vYYYYMMDD-commithash'" + FROM_NIGHTLY_RELEASE=$1 + ;; *) abort "unknown option ${parameter}" ;; esac esac @@ -274,14 +385,24 @@ function parse_flags() { # Do auto release unless release is forced if (( is_auto_release )); then - (( is_dot_release )) && abort "cannot have both --dot-release and --auto-release set simultaneously" - [[ -n "${RELEASE_VERSION}" ]] && abort "cannot have both --version and --auto-release set simultaneously" - [[ -n "${RELEASE_BRANCH}" ]] && abort "cannot have both --branch and --auto-release set simultaneously" - + [[ -n "${RELEASE_VERSION}" ]] && abort "cannot have both --version and --auto-release set simultaneously" + [[ -n "${RELEASE_BRANCH}" ]] && abort "cannot have both --branch and --auto-release set simultaneously" + [[ -n "${FROM_NIGHTLY_RELEASE}" ]] && abort "cannot have --auto-release with a nightly source" setup_upstream prepare_auto_release + fi + # Setup source nightly image + if [[ -n "${FROM_NIGHTLY_RELEASE}" ]]; then + (( is_dot_release )) && abort "dot releases are built from source" + [[ -z "${RELEASE_VERSION}" ]] && abort "release version must be specified with --version" + # TODO(adrcunha): "dot" releases from release branches require releasing nightlies + # for such branches, which we don't do yet. + [[ "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.0$ ]] || abort "version format must be 'X.Y.0'" + RELEASE_BRANCH="release-$(master_version ${RELEASE_VERSION})" + prepare_from_nightly_release + setup_upstream fi # Setup dot releases @@ -302,7 +423,7 @@ function parse_flags() { if (( TAG_RELEASE )); then # Get the commit, excluding any tags but keeping the "dirty" flag local commit="$(git describe --always --dirty --match '^$')" - [[ -n "${commit}" ]] || abort "Error getting the current commit" + [[ -n "${commit}" ]] || abort "error getting the current commit" # Like kubernetes, image tag is vYYYYMMDD-commit TAG="v$(date +%Y%m%d)-${commit}" fi @@ -324,6 +445,7 @@ function parse_flags() { readonly RELEASE_GCS_BUCKET readonly KO_DOCKER_REPO readonly VALIDATION_TESTS + readonly FROM_NIGHTLY_RELEASE } # Run tests (unless --skip-tests was passed). Conveniently displays a banner indicating so. @@ -346,6 +468,9 @@ function main() { parse_flags $@ # Log what will be done and where. banner "Release configuration" + echo "- gcloud user: $(gcloud config get-value core/account)" + echo "- Go path: ${GOPATH}" + echo "- Repository root: ${REPO_ROOT_DIR}" echo "- Destination GCR: ${KO_DOCKER_REPO}" (( SKIP_TESTS )) && echo "- Tests will NOT be run" || echo "- Tests will be run" if (( TAG_RELEASE )); then @@ -361,11 +486,16 @@ function main() { if (( PUBLISH_TO_GITHUB )); then echo "- Release WILL BE published to GitHub" fi - [[ -n "${RELEASE_BRANCH}" ]] && echo "- Release will be built from branch '${RELEASE_BRANCH}'" + if [[ -n "${FROM_NIGHTLY_RELEASE}" ]]; then + echo "- Release will be A COPY OF '${FROM_NIGHTLY_RELEASE}' nightly" + else + echo "- Release will be BUILT FROM SOURCE" + [[ -n "${RELEASE_BRANCH}" ]] && echo "- Release will be built from branch '${RELEASE_BRANCH}'" + fi [[ -n "${RELEASE_NOTES}" ]] && echo "- Release notes are generated from '${RELEASE_NOTES}'" # Checkout specific branch, if necessary - if [[ -n "${RELEASE_BRANCH}" ]]; then + if [[ -n "${RELEASE_BRANCH}" && -z "${FROM_NIGHTLY_RELEASE}" ]]; then setup_upstream setup_branch git checkout upstream/${RELEASE_BRANCH} || abort "cannot checkout branch ${RELEASE_BRANCH}" @@ -374,20 +504,22 @@ function main() { set -o errexit set -o pipefail - run_validation_tests ${VALIDATION_TESTS} - banner "Building the release" - build_release - # Do not use `||` above or any error will be swallowed. - if [[ $? -ne 0 ]]; then - abort "error building the release" + if [[ -n "${FROM_NIGHTLY_RELEASE}" ]]; then + build_from_nightly_release + else + build_from_source fi [[ -z "${YAMLS_TO_PUBLISH}" ]] && abort "no manifests were generated" + # Ensure no empty YAML file will be published. + for yaml in ${YAMLS_TO_PUBLISH}; do + [[ -s ${yaml} ]] || abort "YAML file ${yaml} is empty" + done echo "New release built successfully" if (( PUBLISH_RELEASE )); then tag_images_in_yamls ${YAMLS_TO_PUBLISH} publish_yamls ${YAMLS_TO_PUBLISH} publish_to_github ${YAMLS_TO_PUBLISH} - echo "New release published successfully" + banner "New release published successfully" fi } @@ -410,9 +542,7 @@ function publish_to_github() { cat ${RELEASE_NOTES} >> ${description} fi git tag -a ${TAG} -m "${title}" - local repo_url="${KNATIVE_UPSTREAM}" - [[ -n "${GITHUB_TOKEN}}" ]] && repo_url="${repo_url/:\/\//:\/\/${GITHUB_TOKEN}@}" - hub_tool push ${repo_url} tag ${TAG} + git_push tag ${TAG} [[ -n "${RELEASE_BRANCH}" ]] && commitish="--commitish=${RELEASE_BRANCH}" hub_tool release create \