From 2c098465dd81ade3fbab1dd251f65d2ec4abc1a5 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 24 Aug 2020 16:44:47 -0700 Subject: [PATCH 01/14] Add cleanup instructions to release-notes generation --- SIDECAR_RELEASE_PROCESS.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index f6434c825..91e837c9f 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -54,14 +54,19 @@ naming convention `-on-`. generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. + * Clean up old cached information (also needed if you are generating release + notes for multiple repos) + ```bash + rm -rf /tmp/k8s-repo + ``` * For new minor releases on master: - ``` + ```bash GITHUB_TOKEN= release-notes --discover=mergebase-to-latest --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md ``` * For new patch releases on a release branch: - ``` + ```bash GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md From e73c2ce5312662816b180c5b03eb490322051fd7 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 25 Sep 2020 17:41:33 -0700 Subject: [PATCH 02/14] Use staging registry for canary tests --- prow.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 05d397a97..1812e3a7f 100755 --- a/prow.sh +++ b/prow.sh @@ -211,6 +211,10 @@ configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell functi # still use that name. configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" +# Image registry to use for canary images. +# Only valid if CSI_PROW_DRIVER_CANARY is set. +configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "registry for canary images" + # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) # after setting KUBECONFIG. As a special case, if the repository is Kubernetes, @@ -693,7 +697,7 @@ install_csi_driver () { fi if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then - images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY} IMAGE_REGISTRY=${CSI_PROW_DRIVER_CANARY_REGISTRY}" fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. From d1294628784b0b37b9ce30aceab3725cd32c153e Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 25 Sep 2020 17:49:38 -0700 Subject: [PATCH 03/14] Document new method for adding CI jobs are new K8s versions --- SIDECAR_RELEASE_PROCESS.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 91e837c9f..4575eb819 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -39,10 +39,11 @@ naming convention `-on-`. 1. Changes can then be updated in all the sidecar repos and hostpath driver repo by following the [update instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). -1. New pull and CI jobs are configured by +1. New pull and CI jobs are configured by adding new K8s versions to the top of [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). - New pull jobs that have been unverified should be initially made optional. - [Example](https://github.com/kubernetes/test-infra/pull/15055) + New pull jobs that have been unverified should be initially made optional by + setting the new K8s version as + [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40). 1. Once new pull and CI jobs have been verified, and the new Kubernetes version is released, we can make the optional jobs required, and also remove the Kubernetes versions that are no longer supported. From 7100c1209eba08b3444ad7cb84f31e225e0e3889 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 6 Oct 2020 11:25:19 -0700 Subject: [PATCH 04/14] Only set staging registry when running canary job --- prow.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 1812e3a7f..b069a318f 100755 --- a/prow.sh +++ b/prow.sh @@ -212,7 +212,7 @@ configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell functi configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" # Image registry to use for canary images. -# Only valid if CSI_PROW_DRIVER_CANARY is set. +# Only valid if CSI_PROW_DRIVER_CANARY == "canary". configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "registry for canary images" # The E2E testing can come from an arbitrary repo. The expectation is that @@ -697,7 +697,11 @@ install_csi_driver () { fi if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + if [ "${CSI_PROW_DRIVER_CANARY}" == "canary" ]; then images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY} IMAGE_REGISTRY=${CSI_PROW_DRIVER_CANARY_REGISTRY}" + else + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" + fi fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. From dde93b220ea1c22b56cabc61c1b3057e91653136 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Sat, 26 Sep 2020 19:14:00 +0000 Subject: [PATCH 05/14] Update to snapshot-controller v3.0.0 --- prow.sh | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/prow.sh b/prow.sh index b069a318f..e29adf9f0 100755 --- a/prow.sh +++ b/prow.sh @@ -298,11 +298,7 @@ tests_need_alpha_cluster () { # Regex for non-alpha, feature-tagged tests that should be run. # -# Starting with 1.17, snapshots is beta, but the E2E tests still have the -# [Feature:] tag. They need to be explicitly enabled. -configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing -configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing -configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. @@ -324,7 +320,7 @@ regex_join () { # alpha in previous Kubernetes releases. This was considered too # error prone. Therefore we use E2E tests that match the Kubernetes # version that is getting tested. -configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough +configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for latest Kubernetes version" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests" # After the parallel E2E test without alpha features, a test cluster @@ -339,15 +335,13 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" -configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply @@ -718,7 +712,7 @@ install_csi_driver () { # Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. - CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false @@ -1101,14 +1095,8 @@ main () { start_cluster || die "starting the non-alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller - # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. - if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then - info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" - install_snapshot_crds - install_snapshot_controller - else - info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" - fi + install_snapshot_crds + install_snapshot_controller # Installing the driver might be disabled. if ${CSI_PROW_DRIVER_INSTALL} "$images"; then @@ -1158,14 +1146,8 @@ main () { start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" # Install necessary snapshot CRDs and snapshot controller - # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. - if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then - info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" - install_snapshot_crds - install_snapshot_controller - else - info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" - fi + install_snapshot_crds + install_snapshot_controller # Installing the driver might be disabled. if ${CSI_PROW_DRIVER_INSTALL} "$images"; then From df518fbd6aee7605242b6e66bf6b6e12264750b7 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 16 Oct 2020 12:50:40 +0200 Subject: [PATCH 06/14] prow.sh: usage of Bazel optional Bazel makes sense in the Prow jobs because those often get invoked with a pre-populated Bazel cache. But local invocation don't need it and now can turn it off with CSI_PROW_USE_BAZEL=false. --- prow.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/prow.sh b/prow.sh index e29adf9f0..8c51d232f 100755 --- a/prow.sh +++ b/prow.sh @@ -33,9 +33,9 @@ # The expected environment is: # - $GOPATH/src/ for the repository that is to be tested, # with PR branch merged (when testing a PR) +# - optional: bazel installed (when testing against Kubernetes master), +# must be recent enough for Kubernetes master # - running on linux-amd64 -# - bazel installed (when testing against Kubernetes master), must be recent -# enough for Kubernetes master # - kind (https://github.com/kubernetes-sigs/kind) installed # - optional: Go already installed @@ -111,6 +111,9 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # (if available), otherwise it is built from source. configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" +# Use kind node-image --type=bazel by default, but allow to disable that. +configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" + # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. configvar CSI_PROW_GINKGO_VERSION v1.7.0 "Ginkgo" @@ -537,10 +540,15 @@ start_cluster () { if [ "$version" = "latest" ]; then version=master fi + if ${CSI_PROW_USE_BAZEL}; then + type="bazel" + else + type="docker" + fi git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" - run_with_go "$go_version" kind build node-image --type bazel --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" + run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" From 7bcee13d79012034ece7b46a792c8d2c6160f91a Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 6 Oct 2020 15:33:09 +0200 Subject: [PATCH 07/14] prow.sh: update to kind 0.9, support Kubernetes 1.19 kind 0.9 adds support for recent Kubernetes releases like 1.19 and simplifies configuration of feature gates and runtime config. With Kubernetes 1.19, new feature gates were introduced which might become relevant for the alpha Prow jobs. The updated kind release comes with images for different Kubernetes releases than the one before. To avoid breaking existing jobs, the script now picks kind images automatically. As an additional bonus, it then uses images with hash, i.e. it's less likely to break if those image tags change because of a future kind release. --- prow.sh | 144 ++++++++++++++++++++++---------------------------------- 1 file changed, 55 insertions(+), 89 deletions(-) diff --git a/prow.sh b/prow.sh index 8c51d232f..6186bf946 100755 --- a/prow.sh +++ b/prow.sh @@ -52,26 +52,6 @@ configvar () { eval echo "\$3:" "$1=\${$1}" } -# Takes the minor version of $CSI_PROW_KUBERNETES_VERSION and overrides it to -# $1 if they are equal minor versions. Ignores versions that begin with -# "release-". -override_k8s_version () { - local current_minor_version - local override_minor_version - - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - current_minor_version="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" - - # Ignore: See if you can use ${variable//search/replace} instead. - # shellcheck disable=SC2001 - override_minor_version="$(echo "${1}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1\.\2/')" - if [ "${current_minor_version}" == "${override_minor_version}" ]; then - CSI_PROW_KUBERNETES_VERSION="$1" - echo "Overriding CSI_PROW_KUBERNETES_VERSION with $1: $CSI_PROW_KUBERNETES_VERSION" - fi -} - # Prints the value of a variable + version suffix, falling back to variable + "LATEST". get_versioned_variable () { local var="$1" @@ -107,9 +87,19 @@ configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version fo configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building ginkgo" # depends on CSI_PROW_GINKGO_VERSION below # kind version to use. If the pre-installed version is different, -# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ +# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases # (if available), otherwise it is built from source. -configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" +configvar CSI_PROW_KIND_VERSION "v0.9.0" "kind" + +# kind images to use. Must match the kind version. +# The release notes of each kind release list the supported images. +configvar CSI_PROW_KIND_IMAGES "kindest/node:v1.19.1@sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600 +kindest/node:v1.18.8@sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb +kindest/node:v1.17.11@sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555 +kindest/node:v1.16.15@sha256:a89c771f7de234e6547d43695c7ab047809ffc71a0c3b65aa54eda051c45ed20 +kindest/node:v1.15.12@sha256:d9b939055c1e852fe3d86955ee24976cab46cba518abcb8b13ba70917e6547a6 +kindest/node:v1.14.10@sha256:ce4355398a704fca68006f8a29f37aafb49f8fc2f64ede3ccd0d9198da910146 +kindest/node:v1.13.12@sha256:1c1a48c2bfcbae4d5f4fa4310b5ed10756facad0b7a2ca93c7a4b5bae5db29f5" "kind images" # Use kind node-image --type=bazel by default, but allow to disable that. configvar CSI_PROW_USE_BAZEL true "use Bazel during 'kind node-image' invocation" @@ -127,10 +117,13 @@ configvar CSI_PROW_GINKO_PARALLEL "-p" "Ginko parallelism parameter(s)" configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # Kubernetes version to test against. This must be a version number -# (like 1.13.3) for which there is a pre-built kind image (see -# https://hub.docker.com/r/kindest/node/tags), "latest" (builds -# Kubernetes from the master branch) or "release-x.yy" (builds -# Kubernetes from a release branch). +# (like 1.13.3), "latest" (builds Kubernetes from the master branch) +# or "release-x.yy" (builds Kubernetes from a release branch). +# +# The patch version is only relevant for picking the E2E test suite +# that is used for testing. The script automatically picks +# the kind images for the major/minor version of Kubernetes +# that the kind release supports. # # This can also be a version that was not released yet at the time # that the settings below were chose. The script will then @@ -139,16 +132,6 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # deprecating or changing the implementation of an alpha feature. configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" -# This is a hack to workaround the issue that each version -# of kind currently only supports specific patch versions of -# Kubernetes. We need to override CSI_PROW_KUBERNETES_VERSION -# passed in by our CI/pull jobs to the versions that -# kind v0.5.0 supports. -# -# If the version is prefixed with "release-", then nothing -# is overridden. -override_k8s_version "1.15.3" - # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and # with underscore (1_13 instead of 1.13.3) and in uppercase (LATEST # instead of latest). @@ -338,9 +321,7 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and -# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorageCapacity=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -504,6 +485,22 @@ list_gates () ( done ) +# Turn feature gates in the format foo=true,bar=false into +# a YAML map with the corresponding API groups for use +# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config +list_api_groups () ( + set -f; IFS=',' + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + set -- $1 + while [ "$1" ]; do + if [ "$1" = 'CSIStorageCapacity=true' ]; then + echo ' "storage.k8s.io/v1alpha1": "true"' + fi + shift + done +) + go_version_for_kubernetes () ( local path="$1" local version="$2" @@ -533,8 +530,21 @@ start_cluster () { run kind delete cluster --name=csi-prow || die "kind delete failed" fi - # Build from source? - if [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then + # Try to find a pre-built kind image if asked to use a specific version. + if ! [[ "${CSI_PROW_KUBERNETES_VERSION}" =~ ^release-|^latest$ ]]; then + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + major_minor=$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/^\([0-9]*\)\.\([0-9]*\).*/\1.\2/') + for i in ${CSI_PROW_KIND_IMAGES}; do + if echo "$i" | grep -q "kindest/node:v${major_minor}"; then + image="$i" + break + fi + done + fi + + # Need to build from source? + if ! [ "$image" ]; then if ! ${csi_prow_kind_have_kubernetes}; then local version="${CSI_PROW_KUBERNETES_VERSION}" if [ "$version" = "latest" ]; then @@ -552,63 +562,19 @@ start_cluster () { csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" - else - image="kindest/node:v${CSI_PROW_KUBERNETES_VERSION}" fi cat >"${CSI_PROW_WORK}/kind-config.yaml" <>"${CSI_PROW_WORK}/kind-config.yaml" < Date: Mon, 19 Oct 2020 17:55:37 +0000 Subject: [PATCH 08/14] Add go ldflags using LDFLAGS at the time of compilation --- build.make | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.make b/build.make index c17a1c194..516dc12d7 100644 --- a/build.make +++ b/build.make @@ -69,12 +69,17 @@ endif # toolchain. BUILD_PLATFORMS = +# Add go ldflags using LDFLAGS at the time of compilation. +IMPORTPATH_LDFLAGS = -X main.version=$(REV) +EXT_LDFLAGS = -extldflags "-static" +LDFLAGS = +FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ - if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ From 522361ec9a3e69bd53571f315e4dd2cb5831cb5d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 27 Oct 2020 10:19:18 +0100 Subject: [PATCH 09/14] prow.sh: only run alpha tests for latest Kubernetes release Commit 7bcee13d79012 added alpha feature gates for Kubernetes 1.19 in the CSI_PROW_E2E_ALPHA_GATES_LATEST variable based on the comment in https://github.com/kubernetes-csi/external-provisioner/pull/493#discussion_r502663402 that alpha testing only runs for the latest Kubernetes. But some components (like external-health-monitor) are configured with a single Prow job which runs the default set of tests on a stable Kubernetes release (currently 1.17). Those tests used to include alpha testing which then broke during Kind cluster startup because the Kubernetes 1.19 feature gates weren't recognized by 1.17. The solution is to disable alpha testing for Kubernetes != latest in the default set of tests. --- prow.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 6186bf946..160012ecd 100755 --- a/prow.sh +++ b/prow.sh @@ -248,11 +248,16 @@ configvar CSI_PROW_DEP_VERSION v0.5.1 "golang dep version to be used for vendor # # Unknown or unsupported entries are ignored. # +# Testing of alpha features is only supported for CSI_PROW_KUBERNETES_VERSION=latest +# because CSI_PROW_E2E_ALPHA and CSI_PROW_E2E_ALPHA_GATES are not set for +# older Kubernetes releases. The script supports that, it just isn't done because +# it is not needed and would cause additional maintenance effort. +# # Sanity testing with csi-sanity only covers the CSI driver itself and # thus only makes sense in repos which provide their own CSI # driver. Repos can enable sanity testing by setting # CSI_PROW_TESTS_SANITY=sanity. -configvar CSI_PROW_TESTS "unit parallel serial parallel-alpha serial-alpha sanity" "tests to run" +configvar CSI_PROW_TESTS "unit parallel serial $(if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ]; then echo parallel-alpha serial-alpha; fi) sanity" "tests to run" tests_enabled () { local t1 t2 # We want word-splitting here, so ignore: Quote to prevent word splitting, or split robustly with mapfile or read -a. From 9a370ab902b7be74448753eb63f9e00a0e653bdd Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Tue, 27 Oct 2020 12:19:14 +0100 Subject: [PATCH 10/14] prow.sh: work around "kind build node-image" failure Kind 0.9.0 accesses the "bazel-out" directory with a relative path, which only works when the command is invoked inside the Kubernetes source code directory (https://github.com/kubernetes-sigs/kind/issues/1910). --- prow.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 6186bf946..2f9fb4ff9 100755 --- a/prow.sh +++ b/prow.sh @@ -558,7 +558,8 @@ start_cluster () { git_clone_branch https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version" || die "checking out Kubernetes $version failed" go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" - run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes" || die "'kind build node-image' failed" + # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --type="$type" --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" From 57718f834e8926403455221253c6f31b0ec9aaee Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 12 Nov 2020 14:01:18 +0000 Subject: [PATCH 11/14] Update snapshot CRD version --- prow.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 1ada24803..623477fdc 100755 --- a/prow.sh +++ b/prow.sh @@ -330,7 +330,14 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'GenericEphemeralVolume=true,CSIStorag configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v3.0.0' "external-snapshotter version tag" +default_csi_snapshotter_version () { + if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" == "canary" ]; then + echo "master" + else + echo "v3.0.2" + fi +} +configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply From 79bbca7bc85ce649833e66c66ab0ec3ec5a87d2b Mon Sep 17 00:00:00 2001 From: xing-yang Date: Thu, 12 Nov 2020 19:26:36 +0000 Subject: [PATCH 12/14] Cleanup --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 623477fdc..75eedbdbd 100755 --- a/prow.sh +++ b/prow.sh @@ -331,7 +331,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_ # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment default_csi_snapshotter_version () { - if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" == "canary" ]; then + if [ "${CSI_PROW_KUBERNETES_VERSION}" = "latest" ] || [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo "master" else echo "v3.0.2" From 45ec4c69fbba9291a2aaac4c47eb91149b3db3a5 Mon Sep 17 00:00:00 2001 From: xing-yang Date: Wed, 18 Nov 2020 03:27:57 +0000 Subject: [PATCH 13/14] Fix the install of snapshot CRDs and controller This PR installs snapshot CRDs and rbac rules from the repo and installs snapshot controller from a local image if it is a PR in external-snapshotter repo. Otherwise it uses main or a stable version. --- prow.sh | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/prow.sh b/prow.sh index 75eedbdbd..44511fb23 100755 --- a/prow.sh +++ b/prow.sh @@ -700,6 +700,10 @@ install_csi_driver () { install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + CRD_BASE_DIR="${REPO_DIR}/client/config/crd" + fi + echo "Installing snapshot CRDs from ${CRD_BASE_DIR}" kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false @@ -719,7 +723,16 @@ install_snapshot_crds() { # Install snapshot controller and associated RBAC, retrying until the pod is running. install_snapshot_controller() { - kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + CONTROLLER_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + CONTROLLER_DIR="${REPO_DIR}" + fi + SNAPSHOT_RBAC_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + echo "kubectl apply -f ${SNAPSHOT_RBAC_YAML}" + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + kubectl apply -f ${SNAPSHOT_RBAC_YAML} + cnt=0 until kubectl get clusterrolebinding snapshot-controller-role; do if [ $cnt -gt 30 ]; then @@ -733,8 +746,60 @@ install_snapshot_controller() { sleep 10 done + SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then + # snapshot-controller image built from the PR will get a "csiprow" tag. + # Load it into the "kind" cluster so that we can deploy it. + NEW_TAG="csiprow" + NEW_IMG="snapshot-controller:${NEW_TAG}" + echo "kind load docker-image --name csi-prow ${NEW_IMG}" + kind load docker-image --name csi-prow ${NEW_IMG} || die "could not load the snapshot-controller:csiprow image into the kind cluster" + + # deploy snapshot-controller + echo "Deploying snapshot-controller" + # Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy + # NOTE: This logic is similar to the logic here: + # https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155 + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + # Ignore: Use find instead of ls to better handle non-alphanumeric filenames. + # shellcheck disable=SC2012 + for i in $(ls ${SNAPSHOT_CONTROLLER_YAML} | sort); do + echo " $i" + # Ignore: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead. + # shellcheck disable=SC2002 + # Ignore: See if you can use ${variable//search/replace} instead. + # shellcheck disable=SC2001 + modified="$(cat "$i" | while IFS= read -r line; do + nocomments="$(echo "$line" | sed -e 's/ *#.*$//')" + if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then + # Split 'image: k8s.gcr.io/sig-storage/snapshot-controller:v3.0.0' + # into image (snapshot-controller:v3.0.0), + # name (snapshot-controller), + # tag (v3.0.0). + image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;') + name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;') + tag=$(echo "$image" | sed -e 's;.*:;;') + + # Now replace registry and/or tag + NEW_TAG="csiprow" + line="$(echo "$nocomments" | sed -e "s;$image;${name}:${NEW_TAG};")" + echo " using $line" >&2 + fi + echo "$line" + done)" + if ! echo "$modified" | kubectl apply -f -; then + echo "modified version of $i:" + echo "$modified" + exit 1 + fi + echo "kubectl apply -f ${SNAPSHOT_CONTROLLER_YAML}(modified)" + done + else + echo "kubectl apply -f ${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + kubectl apply -f "${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + fi - kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do From c3a9662513488bd06f1b75a0f6a725f69fc85641 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Tue, 24 Nov 2020 13:02:05 +0000 Subject: [PATCH 14/14] allow export image name and registry name revert part of change --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 516dc12d7..ca2be670a 100644 --- a/build.make +++ b/build.make @@ -20,7 +20,7 @@ # This is the default. It can be overridden in the main Makefile after # including build.make. -REGISTRY_NAME=quay.io/k8scsi +REGISTRY_NAME?=quay.io/k8scsi # Can be set to -mod=vendor to ensure that the "vendor" directory is used. GOFLAGS_VENDOR=