Use a specific template for gMSA

templates/test/ci/prow-ci-version-gmsa/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ../prow-ci-version
+patchesStrategicMerge:
+  - patches/vnet-peering.yaml
+  - patches/machine-identity.yaml

templates/test/ci/prow-ci-version-gmsa/patches/machine-identity.yaml

@@ -0,0 +1,11 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-win
+spec:
+  template:
+    spec:
+      identity: UserAssigned
+      userAssignedIdentities:
+      - providerID: "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-provider-user-identity"
+      - providerID: "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/gmsa-user-identity"

templates/test/ci/prow-ci-version-gmsa/patches/vnet-peering.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  networkSpec:
+    vnet:
+      peerings:
+      - resourceGroup: ${GMSA_NODE_RG}
+        remoteVnetName: dc-${GMSA_ID}-vnet

test/e2e/config/azure-dev.yaml

@@ -161,6 +161,8 @@ providers:
         targetName: "cluster-template-conformance-ci-artifacts-windows-containerd-2022.yaml"
       - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version.yaml"
         targetName: "cluster-template-conformance-ci-artifacts-windows-containerd.yaml"
+      - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-ci-version-gmsa.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts-windows-gmsa-containerd.yaml"
       - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
         targetName: "cluster-template-conformance-presubmit-artifacts.yaml"
       - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml"
@@ -213,7 +215,6 @@ variables:
   SECURITY_SCAN_CONTAINER: "${SECURITY_SCAN_CONTAINER:-quay.io/armosec/kubescape:v1.0.138}"
   # GMSA
   GMSA_KEYVAULT_URL: "https://${CI_RG}-gmsa.vault.azure.net"
-  GMSA_IDENTITY_ID: "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/gmsa-user-identity"
   GMSA_DNS_IP: "${GMSA_DNS_IP}"
   GMSA_ID: "${GMSA_ID}"
 

test/e2e/conformance_test.go

@@ -113,7 +113,7 @@ var _ = Describe("Conformance Tests", func() {
 		kubernetesVersion := e2eConfig.GetVariable(capi_e2e.KubernetesVersion)
 		flavor := clusterctl.DefaultFlavor
 		if isWindows(kubetestConfigFilePath) {
-			flavor = getWindowsFlavor()
+			flavor = getWindowsFlavor(kubetestConfigFilePath)
 		}
 
 		// clusters with CI artifacts or PR artifacts are based on a known CI version
@@ -130,7 +130,7 @@ var _ = Describe("Conformance Tests", func() {
 			}
 
 			if isWindows(kubetestConfigFilePath) {
-				flavor = flavor + "-" + getWindowsFlavor()
+				flavor = flavor + "-" + getWindowsFlavor(kubetestConfigFilePath)
 			}
 		}
 
@@ -269,12 +269,18 @@ var _ = Describe("Conformance Tests", func() {
 // getWindowsFlavor helps choose the correct deployment files. Windows has multiple OS and runtime options that need
 // to be run for conformance.  Current valid options are blank (dockershim) and containerd.  In future will have options
 // for OS version
-func getWindowsFlavor() string {
+func getWindowsFlavor(kubetestConfigFilePath string) string {
+	base := "windows"
+	if strings.Contains(kubetestConfigFilePath, "windows-serial-slow") {
+		base = base + "-gmsa"
+	}
+
 	additionalWindowsFlavor := os.Getenv("WINDOWS_FLAVOR")
 	if additionalWindowsFlavor != "" {
-		return "windows" + "-" + additionalWindowsFlavor
+		return base + "-" + additionalWindowsFlavor
 	}
-	return "windows"
+
+	return base
 }
 
 func isWindows(kubetestConfigFilePath string) bool {

test/e2e/gmsa.go

@@ -30,7 +30,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network"
 	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
-	"github.com/Azure/go-autorest/autorest/to"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
@@ -101,11 +100,10 @@ func configureGmsa(ctx context.Context, workloadProxy, bootstrapClusterProxy fra
 	gmsaNode, windowsNodes := labelGmsaTestNode(ctx, workloadProxy)
 	dropGmsaSpecOnTestNode(gmsaNode, clusterHostName, gmsaSpecFile)
 	configureCoreDNS(ctx, workloadProxy, config)
-	peerDomainVnet(ctx, config, clusterName, subId, networkClient)
 
 	for _, n := range windowsNodes.Items {
 		hostname := getHostName(&n)
-		setUpWorkerNodeIdentities(ctx, vmClient, clusterName, hostname, config)
+		// until https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/2182
 		updateWorkerNodeDNS(config, clusterHostName, hostname)
 	}
 
@@ -122,35 +120,6 @@ func updateWorkerNodeDNS(config *clusterctl.E2EConfig, clusterHostName string, w
 	Expect(err).NotTo(HaveOccurred())
 }
 
-func peerDomainVnet(ctx context.Context, config *clusterctl.E2EConfig, rgName string, subId string, networkClient network.VirtualNetworkPeeringsClient) {
-	gmsaRG := "gmsa-dc-" + config.GetVariable("GMSA_ID")
-	gmsaDomainNetwork := "dc-" + config.GetVariable("GMSA_ID") + "-vnet"
-	clusterVnetName := rgName + "-vnet"
-
-	fmt.Fprintf(GinkgoWriter, "INFO: Peer networks %s\n", config.GetVariable("GMSA_DNS_IP"))
-	gmsaVnetId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", subId, gmsaRG, gmsaDomainNetwork)
-	gmsaPeering := network.VirtualNetworkPeering{
-		VirtualNetworkPeeringPropertiesFormat: &network.VirtualNetworkPeeringPropertiesFormat{
-			RemoteVirtualNetwork: &network.SubResource{
-				ID: to.StringPtr(gmsaVnetId),
-			},
-		},
-	}
-	_, err := networkClient.CreateOrUpdate(ctx, rgName, clusterVnetName, "gmsa-peering", gmsaPeering, network.SyncRemoteAddressSpaceTrue)
-	Expect(err).NotTo(HaveOccurred())
-
-	clusterVnetId := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", subId, rgName, clusterVnetName)
-	clusterPeering := network.VirtualNetworkPeering{
-		VirtualNetworkPeeringPropertiesFormat: &network.VirtualNetworkPeeringPropertiesFormat{
-			RemoteVirtualNetwork: &network.SubResource{
-				ID: to.StringPtr(clusterVnetId),
-			},
-		},
-	}
-	_, err = networkClient.CreateOrUpdate(ctx, gmsaRG, gmsaDomainNetwork, "gmsa-cluster-peering", clusterPeering, network.SyncRemoteAddressSpaceTrue)
-	Expect(err).NotTo(HaveOccurred())
-}
-
 func configureCoreDNS(ctx context.Context, workloadProxy framework.ClusterProxy, config *clusterctl.E2EConfig) {
 	fmt.Fprintf(GinkgoWriter, "INFO: Update coredns with domain ip %s\n", config.GetVariable("GMSA_DNS_IP"))
 
@@ -221,34 +190,6 @@ func labelGmsaTestNode(ctx context.Context, workloadProxy framework.ClusterProxy
 	return gmsaNode, windowsNodes
 }
 
-func setUpWorkerNodeIdentities(ctx context.Context, vmClient compute.VirtualMachinesClient, rgName string, hostname string, config *clusterctl.E2EConfig) {
-	fmt.Fprintf(GinkgoWriter, "INFO: Assigning gmsa identity to cluster vms\n")
-	vm, err := vmClient.Get(ctx, rgName, hostname, "")
-	Expect(err).NotTo(HaveOccurred())
-	existingIdentities := map[string]*compute.VirtualMachineIdentityUserAssignedIdentitiesValue{}
-	if vm.Identity != nil && (*vm.Identity).UserAssignedIdentities != nil {
-		existingIdentities = (*vm.Identity).UserAssignedIdentities
-	}
-
-	gmsaIdentity := config.GetVariable("GMSA_IDENTITY_ID")
-	_, exists := existingIdentities[gmsaIdentity]
-	if !exists {
-		userIdentitiesMap := make(map[string]*compute.VirtualMachineIdentityUserAssignedIdentitiesValue, len(existingIdentities)+1)
-		// copy over existing so we don't overwrite
-		for key, _ := range existingIdentities {
-			userIdentitiesMap[key] = &compute.VirtualMachineIdentityUserAssignedIdentitiesValue{}
-		}
-		// add gmsa identity
-		userIdentitiesMap[gmsaIdentity] = &compute.VirtualMachineIdentityUserAssignedIdentitiesValue{}
-		vmClient.Update(ctx, rgName, *vm.Name, compute.VirtualMachineUpdate{
-			Identity: &compute.VirtualMachineIdentity{
-				Type:                   compute.ResourceIdentityTypeUserAssigned,
-				UserAssignedIdentities: userIdentitiesMap,
-			},
-		})
-	}
-}
-
 func getHostName(gmsaNode *corev1.Node) string {
 	hostname := ""
 	for _, address := range gmsaNode.Status.Addresses {