From f76a955eea34ca688aed4c69d40d72d2463c5bf0 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Sat, 11 Feb 2023 00:57:48 +0000 Subject: [PATCH] Refactor scalesets NIC config --- azure/services/scalesets/scalesets.go | 163 ++++++--------------- azure/services/scalesets/scalesets_test.go | 24 +-- 2 files changed, 59 insertions(+), 128 deletions(-) diff --git a/azure/services/scalesets/scalesets.go b/azure/services/scalesets/scalesets.go index 9ec607d9b1f..0ee2cf2df9b 100644 --- a/azure/services/scalesets/scalesets.go +++ b/azure/services/scalesets/scalesets.go @@ -474,17 +474,6 @@ func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSet diagnosticsProfile := converters.GetDiagnosticsProfile(vmssSpec.DiagnosticsProfile) - // Get the node outbound LB backend pool ID - var backendAddressPools []compute.SubResource - if vmssSpec.PublicLBName != "" { - if vmssSpec.PublicLBAddressPoolName != "" { - backendAddressPools = append(backendAddressPools, - compute.SubResource{ - ID: pointer.String(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), vmssSpec.PublicLBName, vmssSpec.PublicLBAddressPoolName)), - }) - } - } - osProfile, err := s.generateOSProfile(ctx, vmssSpec) if err != nil { return compute.VirtualMachineScaleSet{}, err @@ -509,29 +498,7 @@ func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSet SecurityProfile: securityProfile, DiagnosticsProfile: diagnosticsProfile, NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ - { - Name: pointer.String(vmssSpec.Name), - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: pointer.Bool(true), - EnableIPForwarding: pointer.Bool(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - Name: pointer.String(vmssSpec.Name), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - Subnet: &compute.APIEntityReference{ - ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, vmssSpec.SubnetName)), - }, - Primary: pointer.Bool(true), - PrivateIPAddressVersion: compute.IPVersionIPv4, - LoadBalancerBackendAddressPools: &backendAddressPools, - }, - }, - }, - EnableAcceleratedNetworking: vmssSpec.AcceleratedNetworking, - }, - }, - }, + NetworkInterfaceConfigurations: s.getVirtualMachineScaleSetNetworkConfiguration(vmssSpec), }, Priority: priority, EvictionPolicy: evictionPolicy, @@ -557,71 +524,6 @@ func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSet } } - // Use custom NIC definitions in VMSS if set - if len(vmssSpec.NetworkInterfaces) > 0 { - nicConfigs := []compute.VirtualMachineScaleSetNetworkConfiguration{} - for i, n := range vmssSpec.NetworkInterfaces { - nicConfig := compute.VirtualMachineScaleSetNetworkConfiguration{} - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties = &compute.VirtualMachineScaleSetNetworkConfigurationProperties{} - nicConfig.Name = pointer.String(vmssSpec.Name + "-" + strconv.Itoa(i)) - nicConfig.EnableIPForwarding = pointer.Bool(true) - - if n.AcceleratedNetworking == nil { - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = vmssSpec.AcceleratedNetworking - } else { - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = n.AcceleratedNetworking - } - - if n.PrivateIPConfigs == 0 { - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations = &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - Name: pointer.String(vmssSpec.Name + "-" + strconv.Itoa(i)), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - Subnet: &compute.APIEntityReference{ - ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), - }, - Primary: pointer.Bool(true), - PrivateIPAddressVersion: compute.IPVersionIPv4, - LoadBalancerBackendAddressPools: &backendAddressPools, - }, - }, - } - } else { - ipconfigs := []compute.VirtualMachineScaleSetIPConfiguration{} - - // Create IPConfigs - for j := 0; j < n.PrivateIPConfigs; j++ { - ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ - Name: pointer.String(fmt.Sprintf("private-ipConfig-%v", j)), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - PrivateIPAddressVersion: compute.IPVersionIPv4, - Subnet: &compute.APIEntityReference{ - ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), - }, - }, - } - - ipconfig.Subnet = &compute.APIEntityReference{ - ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), - } - ipconfigs = append(ipconfigs, ipconfig) - } - if i == 0 { - ipconfigs[0].LoadBalancerBackendAddressPools = &backendAddressPools - } - // Always use the first IPConfig as the Primary - ipconfigs[0].Primary = pointer.Bool(true) - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations = &ipconfigs - } - nicConfigs = append(nicConfigs, nicConfig) - } - nicConfigs[0].VirtualMachineScaleSetNetworkConfigurationProperties.Primary = pointer.Bool(true) - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = &nicConfigs - } else { - // Set default interface configuration if no custom ones are specified - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations = s.getVirtualMachineScaleSetDefaultNetworkConfiguration(vmssSpec) - } - // Assign Identity to VMSS if vmssSpec.Identity == infrav1.VMIdentitySystemAssigned { vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ @@ -677,7 +579,7 @@ func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSet return vmss, nil } -func (s *Service) getVirtualMachineScaleSetDefaultNetworkConfiguration(vmssSpec azure.ScaleSetSpec) *[]compute.VirtualMachineScaleSetNetworkConfiguration { +func (s *Service) getVirtualMachineScaleSetNetworkConfiguration(vmssSpec azure.ScaleSetSpec) *[]compute.VirtualMachineScaleSetNetworkConfiguration { var backendAddressPools []compute.SubResource if vmssSpec.PublicLBName != "" { if vmssSpec.PublicLBAddressPoolName != "" { @@ -687,27 +589,50 @@ func (s *Service) getVirtualMachineScaleSetDefaultNetworkConfiguration(vmssSpec }) } } - return &[]compute.VirtualMachineScaleSetNetworkConfiguration{{ - Name: pointer.String(vmssSpec.Name), - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - Primary: pointer.Bool(true), - EnableIPForwarding: pointer.Bool(true), - IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ - { - Name: pointer.String(vmssSpec.Name), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - Subnet: &compute.APIEntityReference{ - ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, vmssSpec.SubnetName)), - }, - Primary: pointer.Bool(true), - PrivateIPAddressVersion: compute.IPVersionIPv4, - LoadBalancerBackendAddressPools: &backendAddressPools, + nicConfigs := []compute.VirtualMachineScaleSetNetworkConfiguration{} + for i, n := range vmssSpec.NetworkInterfaces { + nicConfig := compute.VirtualMachineScaleSetNetworkConfiguration{} + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties = &compute.VirtualMachineScaleSetNetworkConfigurationProperties{} + nicConfig.Name = pointer.String(vmssSpec.Name + "-nic-" + strconv.Itoa(i)) + nicConfig.EnableIPForwarding = pointer.Bool(true) + if n.AcceleratedNetworking != nil { + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = n.AcceleratedNetworking + } else { + // If AcceleratedNetworking is not specified, use the value from the VMSS spec. + // It will be set to true if the VMSS SKU supports it. + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = vmssSpec.AcceleratedNetworking + } + + // Create IPConfigs + ipconfigs := []compute.VirtualMachineScaleSetIPConfiguration{} + for j := 0; j < n.PrivateIPConfigs; j++ { + ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ + Name: pointer.String(fmt.Sprintf("ipConfig" + strconv.Itoa(j))), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPVersionIPv4, + Subnet: &compute.APIEntityReference{ + ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), }, }, - }, - EnableAcceleratedNetworking: vmssSpec.AcceleratedNetworking, - }, - }} + } + + ipconfig.Subnet = &compute.APIEntityReference{ + ID: pointer.String(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), + } + if j == 0 { + // Always use the first IPConfig as the Primary + ipconfig.Primary = pointer.Bool(true) + } + ipconfigs = append(ipconfigs, ipconfig) + } + if i == 0 { + ipconfigs[0].LoadBalancerBackendAddressPools = &backendAddressPools + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.Primary = pointer.Bool(true) + } + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations = &ipconfigs + nicConfigs = append(nicConfigs, nicConfig) + } + return &nicConfigs } // getVirtualMachineScaleSet provides information about a Virtual Machine Scale Set and its instances. diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index c1903812dec..64f7ca9dc18 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -275,11 +275,11 @@ func TestReconcileVMSS(t *testing.T) { setupDefaultVMSSStartCreatingExpectations(s, m) vmss := newDefaultVMSS("VM_SIZE_AN") netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations - (*netConfigs)[0].Name = pointer.String("my-vmss-0") + (*netConfigs)[0].Name = pointer.String("my-vmss-nic-0") (*netConfigs)[0].EnableIPForwarding = pointer.Bool(true) (*netConfigs)[0].EnableAcceleratedNetworking = pointer.Bool(true) nic1IPConfigs := (*netConfigs)[0].IPConfigurations - (*nic1IPConfigs)[0].Name = pointer.String("private-ipConfig-0") + (*nic1IPConfigs)[0].Name = pointer.String("ipConfig0") (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 (*nic1IPConfigs)[0].Subnet = &compute.APIEntityReference{ ID: pointer.String("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/somesubnet"), @@ -321,16 +321,16 @@ func TestReconcileVMSS(t *testing.T) { vmss := newDefaultVMSS("VM_SIZE") vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: pointer.Bool(true)} netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations - (*netConfigs)[0].Name = pointer.String("my-vmss-0") + (*netConfigs)[0].Name = pointer.String("my-vmss-nic-0") (*netConfigs)[0].EnableIPForwarding = pointer.Bool(true) nic1IPConfigs := (*netConfigs)[0].IPConfigurations - (*nic1IPConfigs)[0].Name = pointer.String("private-ipConfig-0") + (*nic1IPConfigs)[0].Name = pointer.String("ipConfig0") (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 (*netConfigs)[0].EnableAcceleratedNetworking = pointer.Bool(true) (*netConfigs)[0].Primary = pointer.Bool(true) vmssIPConfigs := []compute.VirtualMachineScaleSetIPConfiguration{ { - Name: pointer.String("private-ipConfig-0"), + Name: pointer.String("ipConfig0"), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ Primary: pointer.Bool(true), PrivateIPAddressVersion: compute.IPVersionIPv4, @@ -340,7 +340,7 @@ func TestReconcileVMSS(t *testing.T) { }, }, { - Name: pointer.String("private-ipConfig-1"), + Name: pointer.String("ipConfig1"), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ PrivateIPAddressVersion: compute.IPVersionIPv4, Subnet: &compute.APIEntityReference{ @@ -350,7 +350,7 @@ func TestReconcileVMSS(t *testing.T) { }, } *netConfigs = append(*netConfigs, compute.VirtualMachineScaleSetNetworkConfiguration{ - Name: pointer.String("my-vmss-1"), + Name: pointer.String("my-vmss-nic-1"), VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ EnableAcceleratedNetworking: pointer.Bool(true), IPConfigurations: &vmssIPConfigs, @@ -1281,6 +1281,12 @@ func newDefaultVMSSSpec() azure.ScaleSetSpec { AcceleratedNetworking: nil, TerminateNotificationTimeout: pointer.Int(7), FailureDomains: []string{"1", "3"}, + NetworkInterfaces: []infrav1.NetworkInterface{ + { + SubnetName: "my-subnet", + PrivateIPConfigs: 1, + }, + }, } } @@ -1376,14 +1382,14 @@ func newDefaultVMSS(vmSize string) compute.VirtualMachineScaleSet { NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ NetworkInterfaceConfigurations: &[]compute.VirtualMachineScaleSetNetworkConfiguration{ { - Name: pointer.String("my-vmss"), + Name: pointer.String("my-vmss-nic-0"), VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ Primary: pointer.Bool(true), EnableAcceleratedNetworking: pointer.Bool(false), EnableIPForwarding: pointer.Bool(true), IPConfigurations: &[]compute.VirtualMachineScaleSetIPConfiguration{ { - Name: pointer.String("my-vmss"), + Name: pointer.String("ipConfig0"), VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ Subnet: &compute.APIEntityReference{ ID: pointer.String("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet"),