Fix miss match of kube-proxy versions on build

kubernetes-sigs · Jul 30, 2021 · a47dcfe · a47dcfe
1 parent 7f1f6a3
commit a47dcfe
Show file tree

Hide file tree

Showing 11 changed files with 66 additions and 17 deletions.
diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml
@@ -217,4 +217,3 @@ spec:
     namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
   tenantID: ${AZURE_TENANT_ID}
   type: ServicePrincipal
-
diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows.yaml
@@ -29,6 +29,10 @@ spec:
     buildProvenance: ${BUILD_PROVENANCE}
     creationTimestamp: ${TIMESTAMP}
     jobName: ${JOB_NAME}
+  identityRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+    kind: AzureClusterIdentity
+    name: ${CLUSTER_IDENTITY_NAME}
   location: ${AZURE_LOCATION}
   networkSpec:
     vnet:
@@ -161,7 +165,7 @@ spec:
             wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
             $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
             $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
-            $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+            $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
           done
         fi
         echo "* checking binary versions"
@@ -356,7 +360,7 @@ spec:
               wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
               $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
               $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
-              $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+              $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
             done
           fi
           echo "* checking binary versions"
@@ -465,7 +469,7 @@ spec:
           if($$CI_VERSION -ne "")
           {
             $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
-            $$ci_url="https://storage.googleapis.com/kubernetes-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
+            $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
             foreach ( $$binary in $$binaries )
             {
               echo "downloading binary: $$ci_url/$$binary.exe"
@@ -503,6 +507,22 @@ spec:
         sshAuthorizedKeys:
         - ${AZURE_SSH_PUBLIC_KEY:=""}
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+kind: AzureClusterIdentity
+metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
+  name: ${CLUSTER_IDENTITY_NAME}
+  namespace: default
+spec:
+  allowedNamespaces: {}
+  clientID: ${AZURE_CLIENT_ID}
+  clientSecret:
+    name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
+    namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
+  tenantID: ${AZURE_TENANT_ID}
+  type: ServicePrincipal
+---
 apiVersion: addons.cluster.x-k8s.io/v1alpha4
 kind: ClusterResourceSet
 metadata:

diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml
@@ -126,6 +126,7 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    preKubeadmCommands: []
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -215,6 +216,7 @@ spec:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      preKubeadmCommands: []
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
 kind: AzureClusterIdentity

diff --git a/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml b/templates/test/ci/patches/control-plane-kubeadm-boostrap-ci-version.yaml
@@ -54,7 +54,7 @@
           wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
           $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
           $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
-          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
         done
       fi
       echo "* checking binary versions"

diff --git a/templates/test/ci/prow-ci-version-windows/kustomization.yaml b/templates/test/ci/prow-ci-version-windows/kustomization.yaml
@@ -7,7 +7,7 @@ patchesStrategicMerge:
   - patches/windows-image-update.yaml
   - ../patches/control-plane-image-ci-version.yaml
   - ../patches/controller-manager.yaml
-  - ../prow-ci-version/patches/machine-image-ci-version.yaml
+  - ../prow-ci-version/patches/machine-deployment-ci-version.yaml
 patches:
 - target:
     group: bootstrap.cluster.x-k8s.io

diff --git a/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml b/templates/test/ci/prow-ci-version-windows/patches/machine-deployment-ci-version.yaml
@@ -8,7 +8,7 @@
       if($$CI_VERSION -ne "")
       {
         $$binaries=@("kubeadm", "kubectl", "kubelet", "kube-proxy")
-        $$ci_url="https://storage.googleapis.com/kubernetes-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
+        $$ci_url="https://storage.googleapis.com/k8s-release-dev/ci/$$CI_VERSION/bin/windows/amd64"
         foreach ( $$binary in $$binaries )
         {
           echo "downloading binary: $$ci_url/$$binary.exe"

diff --git a/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml b/templates/test/ci/prow-ci-version/patches/kubeadm-bootstrap.yaml
@@ -54,7 +54,7 @@
           wget "$$CI_URL/$$CI_CONTAINER.$$CONTAINER_EXT" -O "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT"
           $${SUDO} ctr -n k8s.io images import "$$CI_DIR/$$CI_CONTAINER.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
           $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" k8s.gcr.io/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
-          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/kubernetes-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
+          $${SUDO} ctr -n k8s.io images tag k8s.gcr.io/$$CI_CONTAINER-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$CI_CONTAINER:"$${CI_VERSION//+/_}"
         done
       fi
       echo "* checking binary versions"

diff --git a/...ion/patches/machine-image-ci-version.yaml → ...atches/machine-deployment-ci-version.yaml b/...ion/patches/machine-image-ci-version.yaml → ...atches/machine-deployment-ci-version.yaml
diff --git a/templates/test/dev/cluster-template-custom-builds-windows.yaml b/templates/test/dev/cluster-template-custom-builds-windows.yaml
@@ -29,6 +29,10 @@ spec:
     buildProvenance: ${BUILD_PROVENANCE}
     creationTimestamp: ${TIMESTAMP}
     jobName: ${JOB_NAME}
+  identityRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+    kind: AzureClusterIdentity
+    name: ${CLUSTER_IDENTITY_NAME}
   location: ${AZURE_LOCATION}
   networkSpec:
     vnet:
@@ -386,13 +390,19 @@ spec:
           }
 
           # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case)
-          # Tag it to the ci version.  The image knows how to use the copy locally.
+          # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION
+          # but matches the kubeproxy image tag when it gets generated. The image configuraiton knows how to use the binary locally.
+          # This does mean the image tage will not match the verison of the binary running.
+          # See:
+          # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60
+          # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59
           docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver
-          docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${KUBE_GIT_VERSION/+/_}-nanoserver"
+          docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver"
 
           kubeadm.exe version -o=short
           kubectl.exe version --client=true --short=true
           kubelet.exe --version
+          kube-proxy.exe --version
         path: C:/replace-k8s-binaries.ps1
         permissions: "0744"
       joinConfiguration:
@@ -415,6 +425,22 @@ spec:
         sshAuthorizedKeys:
         - ${AZURE_SSH_PUBLIC_KEY:=""}
 ---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+kind: AzureClusterIdentity
+metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
+  name: ${CLUSTER_IDENTITY_NAME}
+  namespace: default
+spec:
+  allowedNamespaces: {}
+  clientID: ${AZURE_CLIENT_ID}
+  clientSecret:
+    name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME}
+    namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
+  tenantID: ${AZURE_TENANT_ID}
+  type: ServicePrincipal
+---
 apiVersion: addons.cluster.x-k8s.io/v1alpha4
 kind: ClusterResourceSet
 metadata:

diff --git a/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml b/templates/test/dev/custom-builds-windows/patches/custom-builds-windows.yaml
@@ -13,13 +13,19 @@
       }
 
       # We are using a VHD that maps to v1.18.19 so the kubeproxy image is already pulled. (pull it just in case)
-      # Tag it to the ci version.  The image knows how to use the copy locally.
+      # Tag it to the ci_version which is the version set when kicking off the CI builds and doesn't match the KUBE_GIT_VERSION
+      # but matches the kubeproxy image tag when it gets generated. The image configuraiton knows how to use the binary locally.
+      # This does mean the image tage will not match the verison of the binary running.
+      # See:
+      # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/templates/addons/windows/kube-proxy-windows.yaml#L60
+      # https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/529dbb507962a52ee9fd5a56f3d3856b9bcc53c1/scripts/ci-build-kubernetes.sh#L54-L59
       docker pull sigwindowstools/kube-proxy:v1.18.19-nanoserver
-      docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${KUBE_GIT_VERSION/+/_}-nanoserver"
+      docker tag sigwindowstools/kube-proxy:v1.18.19-nanoserver "sigwindowstools/kube-proxy:${CI_VERSION/+/_}-nanoserver"
 
       kubeadm.exe version -o=short
       kubectl.exe version --client=true --short=true
       kubelet.exe --version
+      kube-proxy.exe --version
     path: C:/replace-k8s-binaries.ps1
     permissions: "0744"
 - op: add

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -32,11 +32,6 @@ import (
 	"testing"
 	"time"
 
-	expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
-	clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
-
-	"github.com/Azure/go-autorest/autorest/to"
-
 	aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
 	"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
@@ -51,6 +46,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
 	expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
+	clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -217,4 +217,3 @@ spec:
		namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}
		tenantID: ${AZURE_TENANT_ID}
		type: ServicePrincipal