From a47dcfe377afbad2088cad7d430f9c1e6edd3024 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Wed, 16 Jun 2021 12:23:01 -0700 Subject: [PATCH] Fix miss match of kube-proxy versions on build --- templates/cluster-template-azure-bastion.yaml | 1 - ...ster-template-prow-ci-version-windows.yaml | 26 ++++++++++++++-- .../ci/cluster-template-prow-custom-vnet.yaml | 2 ++ ...rol-plane-kubeadm-boostrap-ci-version.yaml | 2 +- .../kustomization.yaml | 2 +- .../machine-deployment-ci-version.yaml | 2 +- .../patches/kubeadm-bootstrap.yaml | 2 +- ...aml => machine-deployment-ci-version.yaml} | 0 ...luster-template-custom-builds-windows.yaml | 30 +++++++++++++++++-- .../patches/custom-builds-windows.yaml | 10 +++++-- test/e2e/e2e_suite_test.go | 6 +--- 11 files changed, 66 insertions(+), 17 deletions(-) rename templates/test/ci/prow-ci-version/patches/{machine-image-ci-version.yaml => machine-deployment-ci-version.yaml} (100%) diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index edcb13f4f94..349f85572b0 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -217,4 +217,3 @@ spec: namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} tenantID: ${AZURE_TENANT_ID} type: ServicePrincipal - diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml index 22592c6885b..22806f83f75 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml @@ -29,6 +29,10 @@ spec: buildProvenance: ${BUILD_PROVENANCE} creationTimestamp: ${TIMESTAMP} jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: vnet: @@ -161,7 +165,7 @@ spec: wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" done fi echo "* checking binary versions" @@ -356,7 +360,7 @@ spec: wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" done fi echo "* checking binary versions" @@ -465,7 +469,7 @@ spec: if($$CI_VERSION -ne "") { $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/kubernetes-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" @@ -503,6 +507,22 @@ spec: sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} --- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- apiVersion: addons.cluster.x-k8s.io/v1alpha4 kind: ClusterResourceSet metadata: diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index d6c86437e2c..5d35fa10061 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -126,6 +126,7 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + preKubeadmCommands: [] machineTemplate: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 @@ -215,6 +216,7 @@ spec: cloud-config: /etc/kubernetes/azure.json cloud-provider: azure name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 kind: AzureClusterIdentity diff --git a/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml b/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml index 1fae492ebae..c0a70946a6a 100644 --- a/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml +++ b/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml @@ -54,7 +54,7 @@ wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" done fi echo "* checking binary versions" diff --git a/templates/test/ci/prow-ci-version-windows/kustomization.yaml b/templates/test/ci/prow-ci-version-windows/kustomization.yaml index a7aa8a19bf9..131831af143 100644 --- a/templates/test/ci/prow-ci-version-windows/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-windows/kustomization.yaml @@ -7,7 +7,7 @@ patchesStrategicMerge: - patches/windows-image-update.yaml - ../patches/control-plane-image-ci-version.yaml - ../patches/controller-manager.yaml - - ../prow-ci-version/patches/machine-image-ci-version.yaml + - ../prow-ci-version/patches/machine-deployment-ci-version.yaml patches: - target: group: bootstrap.cluster.x-k8s.io diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml index 7b6010dc8dc..961cbd27ff1 100644 --- a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml +++ b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml @@ -8,7 +8,7 @@ if($$CI_VERSION -ne "") { $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy") - $$ci_url="https://storage.googleapis.com/kubernetes-release-dev/ci/$$CI_VERSION/bin/windows/amd64" + $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64" foreach ( $$binary in $$binaries ) { echo "downloading binary: $$ci_url/$$binary.exe" diff --git a/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml b/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml index 703c049a247..03c55e9301d 100644 --- a/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml +++ b/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml @@ -54,7 +54,7 @@ wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result" $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}" done fi echo "* checking binary versions" diff --git a/templates/test/ci/prow-ci-version/patches/machine-image-ci-version.yaml b/templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version.yaml similarity index 100% rename from templates/test/ci/prow-ci-version/patches/machine-image-ci-version.yaml rename to templates/test/ci/prow-ci-version/patches/machine-deployment-ci-version.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml index 244bb76a33e..ca130e5d252 100644 --- a/templates/test/dev/cluster-template-custom-builds-windows.yaml +++ b/templates/test/dev/cluster-template-custom-builds-windows.yaml @@ -29,6 +29,10 @@ spec: buildProvenance: ${BUILD_PROVENANCE} creationTimestamp: ${TIMESTAMP} jobName: ${JOB_NAME} + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: vnet: @@ -386,13 +390,19 @@ spec: } # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci version. The image knows how to use the copy locally. + # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION + # but matches the kubeproxy image tag when it gets generated. The image configuraiton knows how to use the binary locally. + # This does mean the image tage will not match the verison of the binary running. + # See: + # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60 + # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59 docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${KUBE_GIT_VERSION/+/_}-nanoserver" + docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" kubeadm.exe version -o=short kubectl.exe version --client=true --short=true kubelet.exe --version + kube-proxy.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" joinConfiguration: @@ -415,6 +425,22 @@ spec: sshAuthorizedKeys: - ${AZURE_SSH_PUBLIC_KEY:=""} --- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- apiVersion: addons.cluster.x-k8s.io/v1alpha4 kind: ClusterResourceSet metadata: diff --git a/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml b/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml index 27a5a634eee..c62fe55e4e2 100644 --- a/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml +++ b/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml @@ -13,13 +13,19 @@ } # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case) - # Tag it to the ci version. The image knows how to use the copy locally. + # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION + # but matches the kubeproxy image tag when it gets generated. The image configuraiton knows how to use the binary locally. + # This does mean the image tage will not match the verison of the binary running. + # See: + # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60 + # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59 docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver - docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${KUBE_GIT_VERSION/+/_}-nanoserver" + docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver" kubeadm.exe version -o=short kubectl.exe version --client=true --short=true kubelet.exe --version + kube-proxy.exe --version path: C:/replace-k8s-binaries.ps1 permissions: "0744" - op: add diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ca68bf1102f..e88cdc65a43 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -32,11 +32,6 @@ import ( "testing" "time" - expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4" - clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha4" - - "github.com/Azure/go-autorest/autorest/to" - aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -51,6 +46,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4" + clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha4" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap"