Merge pull request #1018 from CecileRobertMichon/conformance-ci

Add ci version conformance test in conformance test suite

Makefile

@@ -116,6 +116,7 @@ GINKGO_FOCUS ?= Workload cluster creation
 GINKGO_SKIP ?= Creating a GPU-enabled cluster
 GINKGO_NODES ?= 3
 GINKGO_NOCOLOR ?= false
+GINKGO_ARGS ?=
 ARTIFACTS ?= $(ROOT_DIR)/_artifacts
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
@@ -158,37 +159,37 @@ test-cover: envs-test $(KUBECTL) $(KUBE_APISERVER) $(ETCD) ## Run tests with cod
 	go tool cover -func=coverage.out -o coverage.txt
 	go tool cover -html=coverage.out -o coverage.html
 
-.PHONY: test-e2e
-test-e2e: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests
-	PULL_POLICY=IfNotPresent $(MAKE) docker-build docker-push
-	MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
+.PHONY: test-e2e-run
+test-e2e-run: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests
 	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-	$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \
-	    -e2e.artifacts-folder="$(ARTIFACTS)" \
-	    -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-	    -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)
+    $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
+    	-e2e.artifacts-folder="$(ARTIFACTS)" \
+    	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+    	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
 
-.PHONY: test-e2e-local
-test-e2e-local: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests
-	PULL_POLICY=IfNotPresent $(MAKE) docker-build
-	MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
-	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-	$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) -stream --progress ./test/e2e -- \
-	    -e2e.artifacts-folder="$(ARTIFACTS)" \
-	    -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-	    -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)
+.PHONY: test-e2e
+test-e2e: ## Run e2e tests
+	PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
+	$(MAKE) docker-build docker-push \
+	test-e2e-run
 
+LOCAL_GINKGO_ARGS ?= -stream --progress
+LOCAL_GINKGO_ARGS += $(GINKGO_ARGS)
+.PHONY: test-e2e-local
+test-e2e-local: ## Run e2e tests
+	PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
+	$(MAKE) docker-build \
+	GINKGO_ARGS='$(LOCAL_GINKGO_ARGS)' \
+	test-e2e-run
+
+CONFORMANCE_E2E_ARGS ?= -kubetest.config-file=$(KUBETEST_CONF_PATH)
+CONFORMANCE_E2E_ARGS += $(E2E_ARGS)
 .PHONY: test-conformance
-test-conformance: $(ENVSUBST) $(KUBECTL) $(GINKGO) ## Run e2e tests
-	PULL_POLICY=IfNotPresent $(MAKE) docker-build docker-push
-	MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \
-	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-	$(GINKGO) -v -trace -stream -progress -v -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="${GINKGO_SKIP}" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./test/e2e -- \
-	    -e2e.artifacts-folder="$(ARTIFACTS)" \
-	    -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-	    -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \
-			-kubetest.config-file=$(KUBETEST_CONF_PATH) \
-			-e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)
+test-conformance: ## Run conformance test on workload cluster.
+	$(MAKE) test-e2e-local GINKGO_FOCUS="Conformance" E2E_ARGS='$(CONFORMANCE_E2E_ARGS)' GINKGO_ARGS='$(LOCAL_GINKGO_ARGS)'
+
+test-conformance-fast: ## Run conformance test on workload cluster using a subset of the conformance suite in parallel.
+	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_FAST_CONF_PATH) -kubetest.ginkgo-nodes=5 $(E2E_ARGS)"
 
 $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools

templates/test/cluster-template-prow-ci-version.yaml

@@ -405,3 +405,145 @@ spec:
   - kind: ConfigMap
     name: cni-${CLUSTER_NAME}-crs-0
   strategy: ApplyOnce
+---
+apiVersion: exp.cluster.x-k8s.io/v1alpha3
+kind: MachinePool
+metadata:
+  name: ${CLUSTER_NAME}-mp-0
+  namespace: default
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: ${WORKER_MACHINE_COUNT}
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+          kind: KubeadmConfig
+          name: ${CLUSTER_NAME}-mp-0
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
+        kind: AzureMachinePool
+        name: ${CLUSTER_NAME}-mp-0
+      version: ${KUBERNETES_VERSION}
+---
+apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
+kind: AzureMachinePool
+metadata:
+  name: ${CLUSTER_NAME}-mp-0
+  namespace: default
+spec:
+  location: ${AZURE_LOCATION}
+  template:
+    image:
+      marketplace:
+        offer: capi
+        publisher: cncf-upstream
+        sku: k8s-1dot18dot8-ubuntu-1804
+        version: 2020.08.17
+    osDisk:
+      diskSizeGB: 30
+      managedDisk:
+        storageAccountType: Premium_LRS
+      osType: Linux
+    sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
+    vmSize: ${AZURE_NODE_MACHINE_TYPE}
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: KubeadmConfig
+metadata:
+  name: ${CLUSTER_NAME}-mp-0
+  namespace: default
+spec:
+  files:
+  - content: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+      [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+      GSUTIL=gsutil
+      if ! command -v $${GSUTIL} > /dev/null; then
+        curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1
+        GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil
+        # For faster downloads
+        pip install --no-cache-dir -U crcmod
+      fi
+      $${GSUTIL} version
+      # This test installs release packages or binaries that are a result of the CI and release builds.
+      # It runs '... --version' commands to verify that the binaries are correctly installed
+      # and finally uninstalls the packages.
+      # For the release packages it tests all versions in the support skew.
+      LINE_SEPARATOR="*************************************************"
+      echo "$$LINE_SEPARATOR"
+      CI_VERSION=${CI_VERSION}
+      if [[ "$${CI_VERSION}" != "" ]]; then
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p $$CI_DIR
+        declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+        declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+        CONTAINER_EXT="tar"
+        echo "* testing CI version $$CI_VERSION"
+        # Check for semver
+        if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64"
+          VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
+          DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+          curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+          echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
+          apt-get update
+          # replace . with \.
+          VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
+          PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
+            DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
+          done
+        else
+          CI_URL="gs://kubernetes-release-dev/ci-periodic/$$CI_VERSION-bazel/bin/linux/amd64"
+          if ! $${GSUTIL} ls "$${CI_URL}" > /dev/null; then
+            CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64"
+          fi
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
+            $${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE"
+            chmod +x "$$CI_DIR/$$CI_PACKAGE"
+            mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
+          done
+          systemctl restart kubelet
+        fi
+        for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+          echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
+          $${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
+          $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
+          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+        done
+      fi
+      echo "* checking binary versions"
+      echo "ctr version: " $(ctr version)
+      echo "kubeadm version: " $(kubeadm version -o=short)
+      echo "kubectl version: " $(kubectl version --client=true --short=true)
+      echo "kubelet version: " $(kubelet --version)
+      echo "$$LINE_SEPARATOR"
+    owner: root:root
+    path: /tmp/kubeadm-bootstrap.sh
+    permissions: "0744"
+  - contentFrom:
+      secret:
+        key: worker-node-azure.json
+        name: ${CLUSTER_NAME}-mp-0-azure-json
+    owner: root:root
+    path: /etc/kubernetes/azure.json
+    permissions: "0644"
+  joinConfiguration:
+    nodeRegistration:
+      kubeletExtraArgs:
+        cloud-config: /etc/kubernetes/azure.json
+        cloud-provider: azure
+      name: '{{ ds.meta_data["local_hostname"] }}'
+  preKubeadmCommands:
+  - bash -c /tmp/kubeadm-bootstrap.sh
+  useExperimentalRetryJoin: true

templates/test/prow-ci-version/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 namespace: default
 resources:
   - ../prow
+  - patches/machine-pool.yaml
 patchesStrategicMerge:
-  - ../patches/control-plane-image-ci-version.yaml
-  - patches/machine-deployment-ci-version.yaml
+  - patches/azuremachine-marketplace-image.yaml
+  - patches/ci-artifacts-script.yaml

templates/test/prow-ci-version/patches/azuremachine-marketplace-image.yaml

@@ -0,0 +1,46 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: AzureMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  template:
+    spec:
+      image:
+        # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions.
+        # 1.18.8 binaries and images will get replaced to the desired version by preKubeadmCommands.
+        marketplace:
+          publisher: cncf-upstream
+          offer: capi
+          sku: k8s-1dot18dot8-ubuntu-1804
+          version: "2020.08.17"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      image:
+        # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions.
+        # 1.18.8 binaries and images will get replaced to the desired version by preKubeadmCommands.
+        marketplace:
+          publisher: cncf-upstream
+          offer: capi
+          sku: k8s-1dot18dot8-ubuntu-1804
+          version: "2020.08.17"
+---
+apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
+kind: AzureMachinePool
+metadata:
+  name: ${CLUSTER_NAME}-mp-0
+spec:
+  template:
+    image:
+      # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions.
+      # 1.18.8 binaries and images will get replaced to the desired version by the script above.
+      marketplace:
+        publisher: cncf-upstream
+        offer: capi
+        sku: k8s-1dot18dot8-ubuntu-1804
+        version: "2020.08.17"

templates/test/prow-ci-version/patches/machine-deployment-ci-version.yaml → templates/test/prow-ci-version/patches/ci-artifacts-script.yaml

@@ -184,18 +184,92 @@ spec:
               key: control-plane-azure.json
               name: ${CLUSTER_NAME}-control-plane-azure-json
 ---
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
-kind: AzureMachineTemplate
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: KubeadmConfig
 metadata:
-  name: ${CLUSTER_NAME}-md-0
+  name: ${CLUSTER_NAME}-mp-0
 spec:
-  template:
-    spec:
-      image:
-        # we use the 1.18.8 image as a workaround there is no published marketplace image for k8s CI versions.
-        # 1.18.8 binaries and images will get replaced to the desired version by the script above.
-        marketplace:
-          publisher: cncf-upstream
-          offer: capi
-          sku: k8s-1dot18dot8-ubuntu-1804
-          version: "2020.08.17"
+  preKubeadmCommands:
+    - bash -c /tmp/kubeadm-bootstrap.sh
+  files:
+    - path: /tmp/kubeadm-bootstrap.sh
+      owner: "root:root"
+      permissions: "0744"
+      content: |
+        #!/bin/bash
+
+        set -o nounset
+        set -o pipefail
+        set -o errexit
+        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+        GSUTIL=gsutil
+        if ! command -v $${GSUTIL} > /dev/null; then
+          curl -sSL https://sdk.cloud.google.com > /tmp/gcl && bash /tmp/gcl --install-dir=~/gcloud --disable-prompts > /dev/null 2>&1
+          GSUTIL=~/gcloud/google-cloud-sdk/bin/gsutil
+          # For faster downloads
+          pip install --no-cache-dir -U crcmod
+        fi
+        $${GSUTIL} version
+        # This test installs release packages or binaries that are a result of the CI and release builds.
+        # It runs '... --version' commands to verify that the binaries are correctly installed
+        # and finally uninstalls the packages.
+        # For the release packages it tests all versions in the support skew.
+        LINE_SEPARATOR="*************************************************"
+        echo "$$LINE_SEPARATOR"
+        CI_VERSION=${CI_VERSION}
+        if [[ "$${CI_VERSION}" != "" ]]; then
+          CI_DIR=/tmp/k8s-ci
+          mkdir -p $$CI_DIR
+          declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+          declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+          CONTAINER_EXT="tar"
+          echo "* testing CI version $$CI_VERSION"
+          # Check for semver
+          if [[ "$${CI_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            CI_URL="gs://kubernetes-release/release/$$CI_VERSION/bin/linux/amd64"
+            VERSION_WITHOUT_PREFIX="${CI_VERSION#v}"
+            DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+            curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+            echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' > /etc/apt/sources.list.d/kubernetes.list
+            apt-get update
+            # replace . with \.
+            VERSION_REGEX="${VERSION_WITHOUT_PREFIX//./\\.}"
+            PACKAGE_VERSION="$(apt-cache madison kubelet|grep $${VERSION_REGEX}- | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+            for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+              echo "* installing package: $$CI_PACKAGE $${PACKAGE_VERSION}"
+              DEBIAN_FRONTEND=noninteractive apt-get install -y $$CI_PACKAGE=$$PACKAGE_VERSION
+            done
+          else
+            CI_URL="gs://kubernetes-release-dev/ci-periodic/$$CI_VERSION-bazel/bin/linux/amd64"
+            if ! $${GSUTIL} ls "$${CI_URL}" > /dev/null; then
+              CI_URL="gs://kubernetes-release-dev/ci/$$CI_VERSION-bazel/bin/linux/amd64"
+            fi
+            for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+              echo "* downloading binary: $$CI_URL/$$CI_PACKAGE"
+              $${GSUTIL} cp "$$CI_URL/$$CI_PACKAGE" "$$CI_DIR/$$CI_PACKAGE"
+              chmod +x "$$CI_DIR/$$CI_PACKAGE"
+              mv "$$CI_DIR/$$CI_PACKAGE" "/usr/bin/$$CI_PACKAGE"
+            done
+            systemctl restart kubelet
+          fi
+          for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+            echo "* downloading package: $$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT"
+            $${GSUTIL} cp "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
+            $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
+            $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+            $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+          done
+        fi
+        echo "* checking binary versions"
+        echo "ctr version: " $(ctr version)
+        echo "kubeadm version: " $(kubeadm version -o=short)
+        echo "kubectl version: " $(kubectl version --client=true --short=true)
+        echo "kubelet version: " $(kubelet --version)
+        echo "$$LINE_SEPARATOR"
+    - path: /etc/kubernetes/azure.json
+      owner: "root:root"
+      permissions: "0644"
+      contentFrom:
+        secret:
+          key: worker-node-azure.json
+          name: ${CLUSTER_NAME}-mp-0-azure-json