Merge pull request #3916 from fabriziopandini/test-k8s-main

🌱 Add e2e tests for workload cluster with Kubernetes from ci/latest

scripts/ci-e2e-lib.sh

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# capi:buildDockerImages builds all the CAPI (and CAPD) docker images, if not already present locally.
+capi:buildDockerImages () {
+  # Configure provider images generation;
+  # please ensure the generated image name matches image names used in the E2E_CONF_FILE
+  export REGISTRY=gcr.io/k8s-staging-cluster-api
+  export TAG=dev
+  export ARCH=amd64
+  export PULL_POLICY=IfNotPresent
+
+  ## Build all Cluster API provider images, if missing
+  if [[ "$(docker images -q $REGISTRY/cluster-api-controller-amd64:$TAG 2> /dev/null)" == "" ]]; then
+    echo "+ Building CAPI images"
+    make docker-build
+  else
+    echo "+ CAPI images already present in the system, skipping make"
+  fi
+
+  ## Build CAPD provider images, if missing
+  if [[ "$(docker images -q $REGISTRY/capd-manager-amd64:$TAG 2> /dev/null)" == "" ]]; then
+    echo "+ Building CAPD images"
+    make -C test/infrastructure/docker docker-build
+  else
+    echo "+ CAPD images already present in the system, skipping make"
+  fi
+}
+
+# k8s::resolveAllVersions checks all the e2e test variables representing a Kubernetes version,
+# and resolves kubernetes version labels (e.g. latest) to the corresponding version numbers.
+k8s::resolveAllVersions() {
+  if [ -n "${KUBERNETES_VERSION:-}" ]; then
+    k8s::resolveVersion "KUBERNETES_VERSION" "$KUBERNETES_VERSION"
+    export KUBERNETES_VERSION=$resolveVersion
+  fi
+
+  if [ -n "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]; then
+    k8s::resolveVersion "KUBERNETES_VERSION_UPGRADE_TO" "$KUBERNETES_VERSION_UPGRADE_TO"
+    export KUBERNETES_VERSION_UPGRADE_TO=$resolveVersion
+  fi
+
+  if [ -n "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]; then
+    k8s::resolveVersion "KUBERNETES_VERSION_UPGRADE_FROM" "$KUBERNETES_VERSION_UPGRADE_FROM"
+    export KUBERNETES_VERSION_UPGRADE_FROM=$resolveVersion
+  fi
+
+  if [ -n "${BUILD_NODE_IMAGE_TAG:-}" ]; then
+    k8s::resolveVersion "BUILD_NODE_IMAGE_TAG" "$BUILD_NODE_IMAGE_TAG"
+    export BUILD_NODE_IMAGE_TAG=$resolveVersion
+  fi
+}
+
+# k8s::resolveVersion resolves kubernetes version labels (e.g. latest) to the corresponding version numbers.
+# The result will be available in the resolveVersion variable which is accessible from the caller.
+#
+# NOTE: this can't be used for kindest/node images pulled from docker hub, given that there are not guarantees that
+# such images are generated in sync with the Kubernetes release process.
+k8s::resolveVersion() {
+  local variableName=$1
+  local version=$2
+
+  resolveVersion=$version
+  if [[ "$version" =~ ^v ]]; then
+    return
+  fi
+
+  if [[ "$version" =~ ^ci/ ]]; then
+    resolveVersion=$(curl -LsS "http://gcsweb.k8s.io/gcs/kubernetes-release-dev/ci/${version#ci/}.txt")
+  else
+    resolveVersion=$(curl -LsS "http://gcsweb.k8s.io/gcs/kubernetes-release/release/${version}.txt")
+  fi
+  echo "+ $variableName=\"$version\" resolved to \"$resolveVersion\""
+}
+
+# k8s::setBuildVersion sets the build version that will be applied by the Kubernetes build command.
+# the func expect an input parameter defining the version to be used.
+k8s::setBuildVersion() {
+  local version=$1
+  echo "+ Setting version for Kubernetes build to $version"
+
+  local major
+  local minor
+  major=$(echo "${version#v}" | awk '{split($0,a,"."); print a[1]}')
+  minor=$(echo "${version#v}" | awk '{split($0,a,"."); print a[2]}')
+
+  cat > build-version << EOL
+export KUBE_GIT_MAJOR=$major
+export KUBE_GIT_MINOR=$minor
+export KUBE_GIT_VERSION=$version
+export KUBE_GIT_TREE_STATE=clean
+export KUBE_GIT_COMMIT=d34db33f
+EOL
+
+  export KUBE_GIT_VERSION_FILE=$PWD/build-version
+}
+
+# kind::buildNodeImage builds a kindest/node images starting from Kubernetes sources.
+# the func expect an input parameter defining the image tag to be used.
+kind::buildNodeImage() {
+  local version=$1
+  version="${version//+/_}"
+
+  # return early if the image already exists
+  if [[ "$(docker images -q kindest/node:"$version" 2> /dev/null)" != "" ]]; then
+    echo "+ image kindest/node:$version already present in the system, skipping build"
+    return
+  fi
+
+  # sets the build version that will be applied by the Kubernetes build command called during .
+  k8s::setBuildVersion "$1"
+
+  # build the node image
+  echo "+ Building kindest/node:$version"
+  kind build node-image --type docker --image "kindest/node:$version"
+}
+
+# kind:prepullImages pre-pull all the images that will be used in the e2e, thus making
+# the actual test run less sensible to the network speed.
+kind:prepullImages () {
+  # Pulling cert manager images so we can pre-load in kind nodes
+  kind::prepullImage "quay.io/jetstack/cert-manager-cainjector:v0.16.1"
+  kind::prepullImage "quay.io/jetstack/cert-manager-webhook:v0.16.1"
+  kind::prepullImage "quay.io/jetstack/cert-manager-controller:v0.16.1"
+
+  # Pulling kindest/node images used by tests
+  # NB. some of those versions might be the same
+  if [ -n "${KUBERNETES_VERSION:-}" ]; then
+    kind::prepullImage "kindest/node:$KUBERNETES_VERSION"
+  fi
+
+  if [ -n "${KUBERNETES_VERSION_UPGRADE_TO:-}" ]; then
+    kind::prepullImage "kindest/node:$KUBERNETES_VERSION_UPGRADE_TO"
+  fi
+
+  if [ -n "${KUBERNETES_VERSION_UPGRADE_FROM:-}" ]; then
+    kind::prepullImage "kindest/node:$KUBERNETES_VERSION_UPGRADE_FROM"
+  fi
+
+  if [ -n "${BUILD_NODE_IMAGE_TAG:-}" ]; then
+    kind::prepullImage "kindest/node:$BUILD_NODE_IMAGE_TAG"
+  fi
+}
+
+# kind:prepullImage pre-pull a docker image if no already present locally.
+kind::prepullImage () {
+  local image=$1
+  image="${image//+/_}"
+
+  if [[ "$(docker images -q "$image" 2> /dev/null)" == "" ]]; then
+    echo "+ Pulling $image"
+    docker pull "$image"
+  else
+    echo "+ image $image already present in the system, skipping pre-pull"
+  fi
+}

scripts/ci-e2e.sh

@@ -15,43 +15,52 @@
 # limitations under the License.
 
 set -o errexit
-set -o nounset
 set -o pipefail
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
 cd "${REPO_ROOT}" || exit 1
 
+# shellcheck source=./scripts/ci-e2e-lib.sh
+source "${REPO_ROOT}/scripts/ci-e2e-lib.sh"
+
 # shellcheck source=./hack/ensure-go.sh
 source "${REPO_ROOT}/hack/ensure-go.sh"
 # shellcheck source=./hack/ensure-kubectl.sh
 source "${REPO_ROOT}/hack/ensure-kubectl.sh"
 # shellcheck source=./hack/ensure-kustomize.sh
 source "${REPO_ROOT}/hack/ensure-kustomize.sh"
+# shellcheck source=./hack/ensure-kind.sh
+source "${REPO_ROOT}/hack/ensure-kind.sh"
 
 # Make sure the tools binaries are on the path.
 export PATH="${REPO_ROOT}/hack/tools/bin:${PATH}"
 
-# Configure provider images generation;
-# please ensure the generated image name matches image names used in the E2E_CONF_FILE
-export REGISTRY=gcr.io/k8s-staging-cluster-api
-export TAG=dev
-export ARCH=amd64
-export PULL_POLICY=IfNotPresent
-
-## Rebuild all Cluster API provider images
-make docker-build
+# Builds CAPI (and CAPD) images.
+capi:buildDockerImages
 
-## Rebuild CAPD provider images
-make -C test/infrastructure/docker docker-build
+# Checks all the e2e test variables representing a Kubernetes version,
+# and resolves kubernetes version labels (e.g. latest) to the corresponding version numbers.
+# Following variables are currently checked (if defined):
+# - KUBERNETES_VERSION
+# - KUBERNETES_VERSION_UPGRADE_TO
+# - KUBERNETES_VERSION_UPGRADE_FROM
+# - BUILD_NODE_IMAGE_TAG
+k8s::resolveAllVersions
 
-## Pulling cert manager images so we can pre-load in kind nodes
-docker pull quay.io/jetstack/cert-manager-cainjector:v0.16.1
-docker pull quay.io/jetstack/cert-manager-webhook:v0.16.1
-docker pull quay.io/jetstack/cert-manager-controller:v0.16.1
+# If it is required to build a kindest/node image, build it ensuring the generated binary gets
+# the expected version.
+if [ -n "${BUILD_NODE_IMAGE_TAG:-}" ]; then
+  kind::buildNodeImage "$BUILD_NODE_IMAGE_TAG"
+fi
 
-## Pulling kind images used by tests
-docker pull kindest/node:v1.19.1
-docker pull kindest/node:v1.18.2
+# pre-pull all the images that will be used in the e2e, thus making the actual test run
+# less sensible to the network speed. This includes:
+# - cert-manager images
+# - kindest/node:KUBERNETES_VERSION (if defined)
+# - kindest/node:KUBERNETES_VERSION_UPGRADE_TO (if defined)
+# - kindest/node:KUBERNETES_VERSION_UPGRADE_FROM (if defined)
+# - kindest/node:BUILD_NODE_IMAGE_TAG (if defined)
+kind:prepullImages
 
 # Configure e2e tests
 export GINKGO_NODES=3
@@ -64,4 +73,5 @@ export USE_EXISTING_CLUSTER=false
 
 # Run e2e tests
 mkdir -p "$ARTIFACTS"
+echo "+ run tests!"
 make -C test/e2e/ run

test/e2e/config/docker.yaml

@@ -81,6 +81,8 @@ providers:
     - sourcePath: "../data/shared/v1alpha4/metadata.yaml"
 
 variables:
+  # default variables for the e2e test; those values could be overridden via env variables, thus
+  # allowing the same e2e config file to be re-used in different prow jobs e.g. each one with a K8s version permutation
   KUBERNETES_VERSION: "v1.19.1"
   ETCD_VERSION_UPGRADE_TO: "3.4.9-0"
   COREDNS_VERSION_UPGRADE_TO: "1.7.0"

test/e2e/kcp_upgrade_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-var _ = Describe("When testing KCP upgrade", func() {
+var _ = Describe("When testing KCP upgrade [Periodic-K8SVersion]", func() {
 
 	KCPUpgradeSpec(context.TODO(), func() KCPUpgradeSpecInput {
 		return KCPUpgradeSpecInput{

test/e2e/machine_pool_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-var _ = Describe("When testing MachinePools", func() {
+var _ = Describe("When testing MachinePools [Periodic-K8SVersion]", func() {
 	MachinePoolSpec(context.TODO(), func() MachinePoolInput {
 		return MachinePoolInput{
 			E2EConfig:             e2eConfig,

test/e2e/md_upgrades_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-var _ = Describe("When testing MachineDeployment upgrades", func() {
+var _ = Describe("When testing MachineDeployment upgrades [Periodic-K8SVersion]", func() {
 
 	MachineDeploymentUpgradesSpec(context.TODO(), func() MachineDeploymentUpgradesSpecInput {
 		return MachineDeploymentUpgradesSpecInput{

test/e2e/quick_start_test.go

@@ -24,7 +24,7 @@ import (
 	. "github.com/onsi/ginkgo"
 )
 
-var _ = Describe("When following the Cluster API quick-start [PR-Blocking]", func() {
+var _ = Describe("When following the Cluster API quick-start [PR-Blocking] [Periodic-K8SVersion]", func() {
 
 	QuickStartSpec(context.TODO(), func() QuickStartSpecInput {
 		return QuickStartSpecInput{

test/framework/clusterctl/e2e_config.go

@@ -518,11 +518,15 @@ func (c *E2EConfig) GetIntervals(spec, key string) []interface{} {
 	return intervalsInterfaces
 }
 
-// GetVariable returns a variable from the e2e config file.
+// GetVariable returns a variable from environment variables or from the e2e config file.
 func (c *E2EConfig) GetVariable(varName string) string {
-	version, ok := c.Variables[varName]
+	if value, ok := os.LookupEnv(varName); ok {
+		return value
+	}
+
+	value, ok := c.Variables[varName]
 	Expect(ok).NotTo(BeFalse())
-	return version
+	return value
 }
 
 // GetInt64PtrVariable returns an Int64Ptr variable from the e2e config file.

test/framework/clusterctl/repository.go

@@ -121,8 +121,8 @@ func CreateRepository(ctx context.Context, input CreateRepositoryInput) string {
 			"overridesFolder": overridePath,
 		},
 	}
-	for key, value := range input.E2EConfig.Variables {
-		clusterctlConfigFile.Values[key] = value
+	for key := range input.E2EConfig.Variables {
+		clusterctlConfigFile.Values[key] = input.E2EConfig.GetVariable(key)
 	}
 	clusterctlConfigFile.write()
 

test/framework/daemonset_helpers.go

@@ -21,6 +21,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	containerutil "sigs.k8s.io/cluster-api/util/container"
 
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -43,7 +44,7 @@ func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeI
 		if err := input.Getter.Get(ctx, client.ObjectKey{Name: "kube-proxy", Namespace: metav1.NamespaceSystem}, ds); err != nil {
 			return false, err
 		}
-		if ds.Spec.Template.Spec.Containers[0].Image == "k8s.gcr.io/kube-proxy:"+input.KubernetesVersion {
+		if ds.Spec.Template.Spec.Containers[0].Image == "k8s.gcr.io/kube-proxy:"+containerutil.SemverToOCIImageTag(input.KubernetesVersion) {
 			return true, nil
 		}
 		return false, nil

test/framework/machinepool_helpers.go

@@ -129,7 +129,8 @@ func UpgradeMachinePoolAndWait(ctx context.Context, input UpgradeMachinePoolAndW
 	Expect(input.MachinePools).ToNot(BeNil(), "Invalid argument. input.MachinePools can't be empty when calling UpgradeMachinePoolAndWait")
 
 	mgmtClient := input.ClusterProxy.GetClient()
-	for _, mp := range input.MachinePools {
+	for i := range input.MachinePools {
+		mp := input.MachinePools[i]
 		log.Logf("Patching the new kubernetes version to Machine Pool %s/%s", mp.Namespace, mp.Name)
 		patchHelper, err := patch.NewHelper(mp, mgmtClient)
 		Expect(err).ToNot(HaveOccurred())
@@ -138,7 +139,8 @@ func UpgradeMachinePoolAndWait(ctx context.Context, input UpgradeMachinePoolAndW
 		Expect(patchHelper.Patch(ctx, mp)).To(Succeed())
 	}
 
-	for _, mp := range input.MachinePools {
+	for i := range input.MachinePools {
+		mp := input.MachinePools[i]
 		oldVersion := mp.Spec.Template.Spec.Version
 		log.Logf("Waiting for Kubernetes versions of machines in MachinePool %s/%s to be upgraded from %s to %s",
 			mp.Namespace, mp.Name, *oldVersion, input.UpgradeVersion)
@@ -169,7 +171,7 @@ func ScaleMachinePoolAndWait(ctx context.Context, input ScaleMachinePoolAndWaitI
 
 	mgmtClient := input.ClusterProxy.GetClient()
 	for _, mp := range input.MachinePools {
-		log.Logf("Patching the new kubernetes version to Machine Pool %s/%s", mp.Namespace, mp.Name)
+		log.Logf("Patching the replica count in Machine Pool %s/%s", mp.Namespace, mp.Name)
 		patchHelper, err := patch.NewHelper(mp, mgmtClient)
 		Expect(err).ToNot(HaveOccurred())
 

test/infrastructure/docker/docker/machine.go

@@ -32,6 +32,7 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha4"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/cloudinit"
 	"sigs.k8s.io/cluster-api/test/infrastructure/docker/docker/types"
+	"sigs.k8s.io/cluster-api/util/container"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
 	"sigs.k8s.io/kind/pkg/cluster/constants"
@@ -132,6 +133,8 @@ func (m *Machine) IsControlPlane() bool {
 }
 
 // ImageVersion returns the version of the image used or nil if not specified
+// NOTE: Image version might be different from the Kubernetes version, because some characters
+// allowed by semver (e.g. +) can't be used for image tags, so they are replaced with "_".
 func (m *Machine) ImageVersion() string {
 	if m.image == "" {
 		return defaultImageTag
@@ -399,5 +402,7 @@ func (m *Machine) machineImage(version *string) string {
 		versionString = fmt.Sprintf("v%s", versionString)
 	}
 
+	versionString = container.SemverToOCIImageTag(versionString)
+
 	return fmt.Sprintf("%s:%s", defaultImageName, versionString)
 }