diff --git a/Makefile b/Makefile index 783d174b1f5..c84149a3b1e 100644 --- a/Makefile +++ b/Makefile @@ -303,6 +303,7 @@ create-management-cluster: $(ENVSUBST) kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.3/cluster-api-components.yaml # Deploy CAPZ + kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=clusterapi kustomize build config | $(ENVSUBST) | kubectl apply -f - # Wait for CAPI pods @@ -326,7 +327,7 @@ create-workload-cluster: $(ENVSUBST) timeout 300 bash -c "while ! kubectl get secrets | grep $(CLUSTER_NAME)-kubeconfig; do sleep 1; done" # Get kubeconfig and store it locally. kubectl get secrets $(CLUSTER_NAME)-kubeconfig -o json | jq -r .data.value | base64 --decode > ./kubeconfig - timeout 300 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done" + timeout 600 bash -c "while ! kubectl --kubeconfig=./kubeconfig get nodes | grep master; do sleep 1; done" # Deploy calico kubectl --kubeconfig=./kubeconfig apply -f https://docs.projectcalico.org/v3.13/manifests/calico.yaml diff --git a/docs/development.md b/docs/development.md index a68343f7ffd..c317fbdffe0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -17,9 +17,11 @@ - [Modules and dependencies](#modules-and-dependencies) - [Manual Testing](#manual-testing) - [Setting up the environment](#setting-up-the-environment) - - [Building and pushing dev images](#building-and-pushing-dev-images) - - [Customizing the cluster deployment](#customizing-the-cluster-deployment) - - [Creating a test cluster](#creating-a-test-cluster) + - [Creating a dev cluster](#creating-a-dev-cluster) + - [Building and pushing dev images](#building-and-pushing-dev-images) + - [Customizing the cluster deployment](#customizing-the-cluster-deployment) + - [Creating the cluster](#creating-the-cluster) + - [Debugging cluster creation](#debugging-cluster-creation) - [Submitting PRs and testing](#submitting-prs-and-testing) - [Executing unit tests](#executing-unit-tests) - [Automated Testing](#automated-testing) @@ -138,7 +140,7 @@ make create-workload-cluster To delete the cluster: -``` +```bash make delete-workload-cluster ``` @@ -149,27 +151,42 @@ make delete-workload-cluster Your environment must have the Azure credentials as outlined in the [getting started prerequisites section](./getting-started.md#Prerequisites) -#### Building and pushing dev images +#### Creating a dev cluster -1. Login to your container registry using `docker login`. +The steps below are provided in a convenient script in [hack/create-dev-cluster.sh](hack/create-dev-cluster.sh). Be sure to set `REGISTRY=""`, `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRIPTION_ID`, and `AZURE_TENANT_ID` before running. Optionally, you can override the different cluster configuration variables. For example, to override the workload cluster name: - e.g., `docker login quay.io` +```bash +CLUSTER_NAME= ./hack/create-dev-cluster.sh +``` + +##### Building and pushing dev images -2. To build images with custom tags and push to your custom image registry, +1. To build images with custom tags, run the `make docker-build` as follows: ```bash - REGISTRY="" MANAGER_IMAGE_TAG="" make docker-build + export REGISTRY="" + export MANAGER_IMAGE_TAG="" # optional - defaults to `dev`. + PULL_POLICY=IfNotPresent make docker-build ``` -3. Push your docker images: +2. (optional) Push your docker images: + + 2.1. Login to your container registry using `docker login`. + + e.g., `docker login quay.io` + + 2.2. Push to your custom image registry: + ```bash REGISTRY="" MANAGER_IMAGE_TAG="" make docker-push ``` -#### Customizing the cluster deployment + NOTE: `make create-cluster` will fetch the manager image locally and load it onto the kind cluster if it is present. -Here is a list of commonly overridden configuration parameters (the full list is available in `templates/cluster-template.yaml`): +##### Customizing the cluster deployment + +Here is a list of required configuration parameters (the full list is available in `templates/cluster-template.yaml`): ```bash # Cluster settings. @@ -200,11 +217,14 @@ echo "Machine SSH key generated in ${SSH_KEY_FILE}" export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') ``` -#### Creating a test cluster +##### Creating the cluster + +⚠️ Make sure you followed the previous two steps to build the dev image and set the required environment variables before proceding. -Ensure kind has been reset: +Ensure dev environment has been reset: ```bash +make clean make kind-reset ``` @@ -214,9 +234,7 @@ Create the cluster: make create-cluster ``` -These steps above are provided in a convient script in [hack/manual-testing.sh](hack/manual-testing.sh). Be sure to set `REGISTRY=""` and `MANAGER_IMAGE_TAG=""` before running. - -#### debugging cluster creation +#### Debugging cluster creation While cluster build out is running, you can optionally follow the controller logs in a separate window as follows: diff --git a/hack/create-dev-cluster.sh b/hack/create-dev-cluster.sh new file mode 100755 index 00000000000..db59da37977 --- /dev/null +++ b/hack/create-dev-cluster.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Verify the required Environment Variables are present. +: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}" +: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}" +: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}" +: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}" +: "${REGISTRY:?Environment variable empty or not defined.}" + +# Cluster settings. +export CLUSTER_NAME="${CLUSTER_NAME:-capz-test}" +export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet + +# Azure settings. +export AZURE_LOCATION="${AZURE_LOCATION:-southcentralus}" +export AZURE_RESOURCE_GROUP=${CLUSTER_NAME} +export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" +export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" +export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" + +# Machine settings. +export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-3} +export AZURE_CONTROL_PLANE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-Standard_D2s_v3}" +export AZURE_NODE_MACHINE_TYPE="${NODE_MACHINE_TYPE:-Standard_D2s_v3}" +export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-2} +export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.17.3}" + +# Generate SSH key. +SSH_KEY_FILE=${SSH_KEY_FILE:-""} +if ! [ -n "$SSH_KEY_FILE" ]; then + SSH_KEY_FILE=.sshkey + rm -f "${SSH_KEY_FILE}" 2>/dev/null + ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null + echo "Machine SSH key generated in ${SSH_KEY_FILE}" +fi +export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') + +echo "================ DOCKER BUILD ===============" +PULL_POLICY=IfNotPresent make modules docker-build + +echo "================ MAKE CLEAN ===============" +make clean + +echo "================ KIND RESET ===============" +make kind-reset + +echo "================ CREATE CLUSTER ===============" +make create-cluster diff --git a/hack/manual-testing.sh b/hack/manual-testing.sh deleted file mode 100755 index 8e4efee8764..00000000000 --- a/hack/manual-testing.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ -z "${REGISTRY}" ] || [ -z "${MANAGER_IMAGE_TAG}" ]; then - echo "please set environment variables REGISTRY and MANAGER_IMAGE_TAG" - exit 1 -fi - -echo "================ DOCKER BUILD ===============" -make docker-build -echo "================ DOCKER PUSH ===============" -make docker-push - -echo "================ MAKE CLEAN ===============" -make clean - -echo "================ MAKE MANIFESTS ===============" -make generate-manifests -echo "================ MAKE BINARIES ===============" -make binaries -echo "================ KIND RESET ===============" -make kind-reset - -echo "================ CREATE CLUSTER ===============" -make create-cluster diff --git a/scripts/ci-conformance.sh b/scripts/ci-conformance.sh new file mode 100755 index 00000000000..0c57e6c5fd5 --- /dev/null +++ b/scripts/ci-conformance.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### + +# To run locally, set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID + +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# shellcheck source=../hack/ensure-go.sh +source "${REPO_ROOT}/hack/ensure-go.sh" +# shellcheck source=../hack/ensure-kind.sh +source "${REPO_ROOT}/hack/ensure-kind.sh" +# shellcheck source=../hack/ensure-kubectl.sh +source "${REPO_ROOT}/hack/ensure-kubectl.sh" +# shellcheck source=../hack/ensure-kustomize.sh +source "${REPO_ROOT}/hack/ensure-kustomize.sh" + +random-string() { + cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-32} | head -n 1 +} + +# generate manifests needed for creating the Azure cluster to run the tests +add_kustomize_patch() { + # Enable the bits to inject a script that can pull newer versions of kubernetes + if ! grep -i -wq "patchesStrategicMerge" "templates/kustomization.yaml"; then + echo "patchesStrategicMerge:" >> "templates/kustomization.yaml" + fi + if ! grep -i -wq "kustomizeversions" "templates/kustomization.yaml"; then + echo "- kustomizeversions.yaml" >> "templates/kustomization.yaml" + fi +} + +# build Kubernetes E2E binaries +build_k8s() { + # possibly enable bazel build caching before building kubernetes + if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then + create_bazel_cache_rcs.sh || true + fi + + pushd "$(go env GOPATH)/src/k8s.io/kubernetes" + + # make sure we have e2e requirements + bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo + + # ensure the e2e script will find our binaries ... + mkdir -p "${PWD}/_output/bin/" + cp -f "${PWD}/bazel-bin/test/e2e/e2e.test" "${PWD}/_output/bin/e2e.test" + cp -f "${PWD}/bazel-bin/vendor/github.com/onsi/ginkgo/ginkgo/darwin_amd64_stripped/ginkgo" "${PWD}/_output/bin/ginkgo" + export KUBECTL_PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)")/kubectl" + PATH="${KUBECTL_PATH}:${PATH}" + export PATH + + # attempt to release some memory after building + sync || true + echo 1 > /proc/sys/vm/drop_caches || true + + popd +} + +create_cluster() { + export CLUSTER_NAME="capz-conformance-$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 6 ; echo '')" + export CI_VERSION=${CI_VERSION:-$(curl -sSL https://dl.k8s.io/ci/k8s-master.txt)} + export REGISTRY=conformance + ${REPO_ROOT}/hack/create-dev-cluster.sh +} + +run_tests() { + # export the target cluster KUBECONFIG if not already set + export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}" + # ginkgo regexes + SKIP="${SKIP:-}" + FOCUS="${FOCUS:-"\\[Conformance\\]"}" + # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel + if [[ "${PARALLEL:-false}" == "true" ]]; then + export GINKGO_PARALLEL=y + if [[ -z "${SKIP}" ]]; then + SKIP="\\[Serial\\]" + else + SKIP="\\[Serial\\]|${SKIP}" + fi + fi + + # get the number of worker nodes + NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \ + -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \ + | grep -cv "node-role.kubernetes.io/master" )" + + # wait for all the nodes to be ready + kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true + + ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" + + # setting this env prevents ginkg e2e from trying to run provider setup + export KUBERNETES_CONFORMANCE_TEST="y" + # run the tests + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \ + '--provider=skeleton' "--num-nodes=${NUM_NODES}" \ + "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \ + "--report-dir=${ARTIFACTS}" '--disable-log-dump=true') + + unset KUBECONFIG + unset KUBERNETES_CONFORMANCE_TEST +} + +# cleanup all resources we use +cleanup() { + timeout 600 kubectl \ + delete cluster "${CLUSTER_NAME}" || true + timeout 600 kubectl \ + wait --for=delete cluster/"${CLUSTER_NAME}" || true + make kind-reset || true + # clean up e2e.test symlink + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true +} + +# create cluster +SKIP_CREATE_CLUSTER=${SKIP_CREATE_CLUSTER:-""} +if [[ -z "${SKIP_CREATE_CLUSTER}" ]]; then + add_kustomize_patch + create_cluster +fi + +# build k8s binaries and run conformance tests +SKIP_TESTS=${SKIP_TESTS:-""} +if [[ -z "${SKIP_TESTS}" ]]; then + build_k8s + run_tests +fi + +# cleanup +SKIP_CLEANUP=${SKIP_CLEANUP:-""} +if [[ -z "${SKIP_CLEANUP}" ]]; then + cleanup +fi diff --git a/templates/kustomization.yaml b/templates/kustomization.yaml index f732f57be9b..a9662788da8 100644 --- a/templates/kustomization.yaml +++ b/templates/kustomization.yaml @@ -3,4 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - cluster-template.yaml \ No newline at end of file + - cluster-template.yaml diff --git a/templates/kustomizeversions.yaml b/templates/kustomizeversions.yaml new file mode 100644 index 00000000000..a80ab7f5ec8 --- /dev/null +++ b/templates/kustomizeversions.yaml @@ -0,0 +1,211 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + clusterConfiguration: + kubernetesVersion: "ci/${CI_VERSION}" + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + files: + - path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + GSUTIL=gsutil + if ! command -v $${GSUTIL} > /dev/null; then + curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1 + GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil + # For faster downloads + pip install --no-cache-dir -U crcmod + fi + $${GSUTIL} version + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + $${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + $${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + - path: /etc/kubernetes/azure.json + owner: "root:root" + permissions: "0644" + content: | + { + "cloud": "AzurePublicCloud", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${AZURE_RESOURCE_GROUP}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "standard", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${CLUSTER_NAME}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "userAssignedID": "${CLUSTER_NAME}", + "loadBalancerSku": "standard", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + preKubeadmCommands: + - bash -c /tmp/kubeadm-bootstrap.sh + files: + - path: /tmp/kubeadm-bootstrap.sh + owner: "root:root" + permissions: "0744" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + GSUTIL=gsutil + if ! command -v $${GSUTIL} > /dev/null; then + curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1 + GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil + # For faster downloads + pip install --no-cache-dir -U crcmod + fi + $${GSUTIL} version + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$$LINE_SEPARATOR" + CI_VERSION=${CI_VERSION} + if [[ "$${CI_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p $$CI_DIR + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $$CI_VERSION" + # Check for semver + if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="${CI_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION + done + else + CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $$CI_URL/$$CI_PACKAGE" + $${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE" + chmod +x "$$CI_DIR/$$CI_PACKAGE" + mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" + $${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" + $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " $(ctr version) + echo "kubeadm version: " $(kubeadm version -o=short) + echo "kubectl version: " $(kubectl version --client=true --short=true) + echo "kubelet version: " $(kubelet --version) + echo "$$LINE_SEPARATOR" + - path: /etc/kubernetes/azure.json + owner: "root:root" + permissions: "0644" + content: | + { + "cloud": "AzurePublicCloud", + "tenantId": "${AZURE_TENANT_ID}", + "subscriptionId": "${AZURE_SUBSCRIPTION_ID}", + "aadClientId": "${AZURE_CLIENT_ID}", + "aadClientSecret": "${AZURE_CLIENT_SECRET}", + "resourceGroup": "${CLUSTER_NAME}", + "securityGroupName": "${CLUSTER_NAME}-node-nsg", + "location": "${AZURE_LOCATION}", + "vmType": "standard", + "vnetName": "${CLUSTER_NAME}-vnet", + "vnetResourceGroup": "${CLUSTER_NAME}", + "subnetName": "${CLUSTER_NAME}-node-subnet", + "routeTableName": "${CLUSTER_NAME}-node-routetable", + "loadBalancerSku": "standard", + "maximumLoadBalancerRuleCount": 250, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } \ No newline at end of file diff --git a/test/e2e/resource_generators.go b/test/e2e/resource_generators.go index f1a30c3bb2a..cda25de8812 100644 --- a/test/e2e/resource_generators.go +++ b/test/e2e/resource_generators.go @@ -80,7 +80,7 @@ var ( ) func (c *ClusterGenerator) GenerateCluster(namespace string) (*capiv1.Cluster, *infrav1.AzureCluster) { - name := "capz-" + util.RandomString(6) + name := "capz-e2e" + util.RandomString(6) vnetName := name + "-vnet" tags := map[string]string{ "creationTimestamp": time.Now().UTC().Format(time.RFC3339),