From 12032805ca563966adb6ef561d439928151d19a4 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Tue, 22 Sep 2020 14:01:09 -0700 Subject: [PATCH 1/7] Enable configurable API Server Load Balancer update templates --- api/v1alpha2/azurecluster_conversion.go | 5 - api/v1alpha2/zz_generated.conversion.go | 188 +------------ api/v1alpha3/azurecluster_default.go | 71 ++++- api/v1alpha3/azurecluster_default_test.go | 56 ++++ api/v1alpha3/azurecluster_types.go | 2 - api/v1alpha3/azurecluster_validation.go | 118 ++++++-- api/v1alpha3/azurecluster_validation_test.go | 255 +++++++++++++++--- api/v1alpha3/azurecluster_webhook.go | 8 +- api/v1alpha3/tags.go | 3 - api/v1alpha3/types.go | 74 ++--- api/v1alpha3/zz_generated.deepcopy.go | 75 ++---- cloud/defaults.go | 26 +- cloud/interfaces.go | 2 + cloud/mocks/service_mock.go | 56 ++++ cloud/scope/cluster.go | 168 ++++++++---- cloud/scope/machine.go | 19 +- cloud/scope/machinepool.go | 4 +- .../mocks_bastionhosts/bastionhosts_mock.go | 28 ++ cloud/services/loadbalancers/loadbalancers.go | 177 +++++------- .../mock_loadbalancers/loadbalancers_mock.go | 28 ++ .../networkinterfaces/networkinterfaces.go | 26 +- cloud/services/publicips/publicips.go | 4 +- .../mock_routetables/routetables_mock.go | 28 ++ .../securitygroups_mock.go | 28 ++ .../subnets/mock_subnets/subnets_mock.go | 28 ++ cloud/services/subnets/subnets.go | 9 +- cloud/types.go | 58 ++-- ...ucture.cluster.x-k8s.io_azureclusters.yaml | 91 +++---- controllers/azurecluster_controller.go | 8 +- controllers/azurecluster_reconciler.go | 21 +- hack/debugging/kubectl-capz-ssh | 2 +- templates/cluster-template.yaml | 199 -------------- .../prow-private/patches/custom-vnet.yaml | 23 ++ 33 files changed, 1017 insertions(+), 871 deletions(-) delete mode 100644 templates/cluster-template.yaml create mode 100644 templates/test/prow-private/patches/custom-vnet.yaml diff --git a/api/v1alpha2/azurecluster_conversion.go b/api/v1alpha2/azurecluster_conversion.go index 866e7019cd0..63ffaaafb4d 100644 --- a/api/v1alpha2/azurecluster_conversion.go +++ b/api/v1alpha2/azurecluster_conversion.go @@ -162,11 +162,6 @@ func Convert_v1alpha3_AzureClusterStatus_To_v1alpha2_AzureClusterStatus(in *infr return nil } -// Convert_v1alpha2_Network_To_v1alpha3_Network. -func Convert_v1alpha2_Network_To_v1alpha3_Network(in *Network, out *infrav1alpha3.Network, s apiconversion.Scope) error { //nolint - return autoConvert_v1alpha2_Network_To_v1alpha3_Network(in, out, s) -} - // Convert_v1alpha2_NetworkSpec_To_v1alpha3_NetworkSpec. func Convert_v1alpha2_NetworkSpec_To_v1alpha3_NetworkSpec(in *NetworkSpec, out *infrav1alpha3.NetworkSpec, s apiconversion.Scope) error { //nolint if err := Convert_v1alpha2_VnetSpec_To_v1alpha3_VnetSpec(&in.Vnet, &out.Vnet, s); err != nil { diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 67a7b856f2b..52e1f428441 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -126,16 +126,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*BackendPool)(nil), (*v1alpha3.BackendPool)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_BackendPool_To_v1alpha3_BackendPool(a.(*BackendPool), b.(*v1alpha3.BackendPool), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1alpha3.BackendPool)(nil), (*BackendPool)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_BackendPool_To_v1alpha2_BackendPool(a.(*v1alpha3.BackendPool), b.(*BackendPool), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*BuildParams)(nil), (*v1alpha3.BuildParams)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_BuildParams_To_v1alpha3_BuildParams(a.(*BuildParams), b.(*v1alpha3.BuildParams), scope) }); err != nil { @@ -146,51 +136,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*FrontendIPConfig)(nil), (*v1alpha3.FrontendIPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig(a.(*FrontendIPConfig), b.(*v1alpha3.FrontendIPConfig), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1alpha3.FrontendIPConfig)(nil), (*FrontendIPConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig(a.(*v1alpha3.FrontendIPConfig), b.(*FrontendIPConfig), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*LoadBalancer)(nil), (*v1alpha3.LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer(a.(*LoadBalancer), b.(*v1alpha3.LoadBalancer), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1alpha3.LoadBalancer)(nil), (*LoadBalancer)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer(a.(*v1alpha3.LoadBalancer), b.(*LoadBalancer), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*ManagedDisk)(nil), (*v1alpha3.ManagedDisk)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_ManagedDisk_To_v1alpha3_ManagedDisk(a.(*ManagedDisk), b.(*v1alpha3.ManagedDisk), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1alpha3.Network)(nil), (*Network)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_Network_To_v1alpha2_Network(a.(*v1alpha3.Network), b.(*Network), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*OSDisk)(nil), (*v1alpha3.OSDisk)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_OSDisk_To_v1alpha3_OSDisk(a.(*OSDisk), b.(*v1alpha3.OSDisk), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*PublicIP)(nil), (*v1alpha3.PublicIP)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_PublicIP_To_v1alpha3_PublicIP(a.(*PublicIP), b.(*v1alpha3.PublicIP), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1alpha3.PublicIP)(nil), (*PublicIP)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha3_PublicIP_To_v1alpha2_PublicIP(a.(*v1alpha3.PublicIP), b.(*PublicIP), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*VM)(nil), (*v1alpha3.VM)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_VM_To_v1alpha3_VM(a.(*VM), b.(*v1alpha3.VM), scope) }); err != nil { @@ -241,11 +196,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddConversionFunc((*Network)(nil), (*v1alpha3.Network)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1alpha2_Network_To_v1alpha3_Network(a.(*Network), b.(*v1alpha3.Network), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*SecurityGroup)(nil), (*v1alpha3.SecurityGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_SecurityGroup_To_v1alpha3_SecurityGroup(a.(*SecurityGroup), b.(*v1alpha3.SecurityGroup), scope) }); err != nil { @@ -438,9 +388,7 @@ func autoConvert_v1alpha3_AzureClusterSpec_To_v1alpha2_AzureClusterSpec(in *v1al } func autoConvert_v1alpha2_AzureClusterStatus_To_v1alpha3_AzureClusterStatus(in *AzureClusterStatus, out *v1alpha3.AzureClusterStatus, s conversion.Scope) error { - if err := Convert_v1alpha2_Network_To_v1alpha3_Network(&in.Network, &out.Network, s); err != nil { - return err - } + // WARNING: in.Network requires manual conversion: does not exist in peer-type // WARNING: in.Bastion requires manual conversion: does not exist in peer-type out.Ready = in.Ready // WARNING: in.APIEndpoints requires manual conversion: does not exist in peer-type @@ -448,9 +396,6 @@ func autoConvert_v1alpha2_AzureClusterStatus_To_v1alpha3_AzureClusterStatus(in * } func autoConvert_v1alpha3_AzureClusterStatus_To_v1alpha2_AzureClusterStatus(in *v1alpha3.AzureClusterStatus, out *AzureClusterStatus, s conversion.Scope) error { - if err := Convert_v1alpha3_Network_To_v1alpha2_Network(&in.Network, &out.Network, s); err != nil { - return err - } // WARNING: in.FailureDomains requires manual conversion: does not exist in peer-type out.Ready = in.Ready // WARNING: in.Conditions requires manual conversion: does not exist in peer-type @@ -725,28 +670,6 @@ func Convert_v1alpha3_AzureMachineTemplateSpec_To_v1alpha2_AzureMachineTemplateS return autoConvert_v1alpha3_AzureMachineTemplateSpec_To_v1alpha2_AzureMachineTemplateSpec(in, out, s) } -func autoConvert_v1alpha2_BackendPool_To_v1alpha3_BackendPool(in *BackendPool, out *v1alpha3.BackendPool, s conversion.Scope) error { - out.Name = in.Name - out.ID = in.ID - return nil -} - -// Convert_v1alpha2_BackendPool_To_v1alpha3_BackendPool is an autogenerated conversion function. -func Convert_v1alpha2_BackendPool_To_v1alpha3_BackendPool(in *BackendPool, out *v1alpha3.BackendPool, s conversion.Scope) error { - return autoConvert_v1alpha2_BackendPool_To_v1alpha3_BackendPool(in, out, s) -} - -func autoConvert_v1alpha3_BackendPool_To_v1alpha2_BackendPool(in *v1alpha3.BackendPool, out *BackendPool, s conversion.Scope) error { - out.Name = in.Name - out.ID = in.ID - return nil -} - -// Convert_v1alpha3_BackendPool_To_v1alpha2_BackendPool is an autogenerated conversion function. -func Convert_v1alpha3_BackendPool_To_v1alpha2_BackendPool(in *v1alpha3.BackendPool, out *BackendPool, s conversion.Scope) error { - return autoConvert_v1alpha3_BackendPool_To_v1alpha2_BackendPool(in, out, s) -} - func autoConvert_v1alpha2_BuildParams_To_v1alpha3_BuildParams(in *BuildParams, out *v1alpha3.BuildParams, s conversion.Scope) error { out.Lifecycle = v1alpha3.ResourceLifecycle(in.Lifecycle) out.ClusterName = in.ClusterName @@ -777,24 +700,6 @@ func Convert_v1alpha3_BuildParams_To_v1alpha2_BuildParams(in *v1alpha3.BuildPara return autoConvert_v1alpha3_BuildParams_To_v1alpha2_BuildParams(in, out, s) } -func autoConvert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig(in *FrontendIPConfig, out *v1alpha3.FrontendIPConfig, s conversion.Scope) error { - return nil -} - -// Convert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig is an autogenerated conversion function. -func Convert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig(in *FrontendIPConfig, out *v1alpha3.FrontendIPConfig, s conversion.Scope) error { - return autoConvert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig(in, out, s) -} - -func autoConvert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig(in *v1alpha3.FrontendIPConfig, out *FrontendIPConfig, s conversion.Scope) error { - return nil -} - -// Convert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig is an autogenerated conversion function. -func Convert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig(in *v1alpha3.FrontendIPConfig, out *FrontendIPConfig, s conversion.Scope) error { - return autoConvert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig(in, out, s) -} - func autoConvert_v1alpha2_Image_To_v1alpha3_Image(in *Image, out *v1alpha3.Image, s conversion.Scope) error { // WARNING: in.Publisher requires manual conversion: does not exist in peer-type // WARNING: in.Offer requires manual conversion: does not exist in peer-type @@ -837,44 +742,6 @@ func autoConvert_v1alpha3_IngressRule_To_v1alpha2_IngressRule(in *v1alpha3.Ingre return nil } -func autoConvert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer(in *LoadBalancer, out *v1alpha3.LoadBalancer, s conversion.Scope) error { - out.ID = in.ID - out.Name = in.Name - out.SKU = v1alpha3.SKU(in.SKU) - if err := Convert_v1alpha2_FrontendIPConfig_To_v1alpha3_FrontendIPConfig(&in.FrontendIPConfig, &out.FrontendIPConfig, s); err != nil { - return err - } - if err := Convert_v1alpha2_BackendPool_To_v1alpha3_BackendPool(&in.BackendPool, &out.BackendPool, s); err != nil { - return err - } - out.Tags = *(*v1alpha3.Tags)(unsafe.Pointer(&in.Tags)) - return nil -} - -// Convert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer is an autogenerated conversion function. -func Convert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer(in *LoadBalancer, out *v1alpha3.LoadBalancer, s conversion.Scope) error { - return autoConvert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer(in, out, s) -} - -func autoConvert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer(in *v1alpha3.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { - out.ID = in.ID - out.Name = in.Name - out.SKU = SKU(in.SKU) - if err := Convert_v1alpha3_FrontendIPConfig_To_v1alpha2_FrontendIPConfig(&in.FrontendIPConfig, &out.FrontendIPConfig, s); err != nil { - return err - } - if err := Convert_v1alpha3_BackendPool_To_v1alpha2_BackendPool(&in.BackendPool, &out.BackendPool, s); err != nil { - return err - } - out.Tags = *(*Tags)(unsafe.Pointer(&in.Tags)) - return nil -} - -// Convert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer is an autogenerated conversion function. -func Convert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer(in *v1alpha3.LoadBalancer, out *LoadBalancer, s conversion.Scope) error { - return autoConvert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer(in, out, s) -} - func autoConvert_v1alpha2_ManagedDisk_To_v1alpha3_ManagedDisk(in *ManagedDisk, out *v1alpha3.ManagedDisk, s conversion.Scope) error { out.StorageAccountType = in.StorageAccountType return nil @@ -891,32 +758,6 @@ func autoConvert_v1alpha3_ManagedDisk_To_v1alpha2_ManagedDisk(in *v1alpha3.Manag return nil } -func autoConvert_v1alpha2_Network_To_v1alpha3_Network(in *Network, out *v1alpha3.Network, s conversion.Scope) error { - // WARNING: in.SecurityGroups requires manual conversion: does not exist in peer-type - if err := Convert_v1alpha2_LoadBalancer_To_v1alpha3_LoadBalancer(&in.APIServerLB, &out.APIServerLB, s); err != nil { - return err - } - if err := Convert_v1alpha2_PublicIP_To_v1alpha3_PublicIP(&in.APIServerIP, &out.APIServerIP, s); err != nil { - return err - } - return nil -} - -func autoConvert_v1alpha3_Network_To_v1alpha2_Network(in *v1alpha3.Network, out *Network, s conversion.Scope) error { - if err := Convert_v1alpha3_LoadBalancer_To_v1alpha2_LoadBalancer(&in.APIServerLB, &out.APIServerLB, s); err != nil { - return err - } - if err := Convert_v1alpha3_PublicIP_To_v1alpha2_PublicIP(&in.APIServerIP, &out.APIServerIP, s); err != nil { - return err - } - return nil -} - -// Convert_v1alpha3_Network_To_v1alpha2_Network is an autogenerated conversion function. -func Convert_v1alpha3_Network_To_v1alpha2_Network(in *v1alpha3.Network, out *Network, s conversion.Scope) error { - return autoConvert_v1alpha3_Network_To_v1alpha2_Network(in, out, s) -} - func autoConvert_v1alpha2_NetworkSpec_To_v1alpha3_NetworkSpec(in *NetworkSpec, out *v1alpha3.NetworkSpec, s conversion.Scope) error { if err := Convert_v1alpha2_VnetSpec_To_v1alpha3_VnetSpec(&in.Vnet, &out.Vnet, s); err != nil { return err @@ -952,6 +793,7 @@ func autoConvert_v1alpha3_NetworkSpec_To_v1alpha2_NetworkSpec(in *v1alpha3.Netwo } else { out.Subnets = nil } + // WARNING: in.APIServerLB requires manual conversion: does not exist in peer-type return nil } @@ -980,32 +822,6 @@ func autoConvert_v1alpha3_OSDisk_To_v1alpha2_OSDisk(in *v1alpha3.OSDisk, out *OS return nil } -func autoConvert_v1alpha2_PublicIP_To_v1alpha3_PublicIP(in *PublicIP, out *v1alpha3.PublicIP, s conversion.Scope) error { - out.ID = in.ID - out.Name = in.Name - out.IPAddress = in.IPAddress - out.DNSName = in.DNSName - return nil -} - -// Convert_v1alpha2_PublicIP_To_v1alpha3_PublicIP is an autogenerated conversion function. -func Convert_v1alpha2_PublicIP_To_v1alpha3_PublicIP(in *PublicIP, out *v1alpha3.PublicIP, s conversion.Scope) error { - return autoConvert_v1alpha2_PublicIP_To_v1alpha3_PublicIP(in, out, s) -} - -func autoConvert_v1alpha3_PublicIP_To_v1alpha2_PublicIP(in *v1alpha3.PublicIP, out *PublicIP, s conversion.Scope) error { - out.ID = in.ID - out.Name = in.Name - out.IPAddress = in.IPAddress - out.DNSName = in.DNSName - return nil -} - -// Convert_v1alpha3_PublicIP_To_v1alpha2_PublicIP is an autogenerated conversion function. -func Convert_v1alpha3_PublicIP_To_v1alpha2_PublicIP(in *v1alpha3.PublicIP, out *PublicIP, s conversion.Scope) error { - return autoConvert_v1alpha3_PublicIP_To_v1alpha2_PublicIP(in, out, s) -} - func autoConvert_v1alpha2_SecurityGroup_To_v1alpha3_SecurityGroup(in *SecurityGroup, out *v1alpha3.SecurityGroup, s conversion.Scope) error { out.ID = in.ID out.Name = in.Name diff --git a/api/v1alpha3/azurecluster_default.go b/api/v1alpha3/azurecluster_default.go index 6903512f48d..cfaed82388f 100644 --- a/api/v1alpha3/azurecluster_default.go +++ b/api/v1alpha3/azurecluster_default.go @@ -27,6 +27,8 @@ const ( DefaultControlPlaneSubnetCIDR = "10.0.0.0/16" // DefaultNodeSubnetCIDR is the default Node Subnet CIDR DefaultNodeSubnetCIDR = "10.1.0.0/16" + // DefaultInternalLBIPAddress is the default internal load balancer ip address + DefaultInternalLBIPAddress = "10.0.0.100" ) const ( @@ -36,16 +38,19 @@ const ( DefaultControlPlaneSubnetIPv6CIDR = "2001:1234:5678:9abc::/64" // DefaultNodeSubnetIPv6CIDR is the default Node Subnet CIDR DefaultNodeSubnetIPv6CIDR = "2001:1234:5678:9abd::/64" + // DefaultInternalLBIPv6Address is the default internal load balancer ip address + DefaultInternalLBIPv6Address = "2001:1234:5678:9abc::100" ) func (c *AzureCluster) setDefaults() { + c.setResourceGroupDefault() c.setNetworkSpecDefaults() } func (c *AzureCluster) setNetworkSpecDefaults() { - c.setResourceGroupDefault() c.setVnetDefaults() c.setSubnetDefaults() + c.setAPIServerLBDefaults() } func (c *AzureCluster) setResourceGroupDefault() { @@ -103,6 +108,50 @@ func (c *AzureCluster) setSubnetDefaults() { } } +func (c *AzureCluster) setAPIServerLBDefaults() { + lb := &c.Spec.NetworkSpec.APIServerLB + if lb.Type == "" { + lb.Type = Public + } + if lb.SKU == "" { + lb.SKU = SKUStandard + } + + if lb.Type == Public { + if lb.Name == "" { + lb.Name = generatePublicLBName(c.ObjectMeta.Name) + } + if len(lb.FrontendIPs) == 0 { + lb.FrontendIPs = []FrontendIP{ + { + Name: generateFrontendIPConfigName(lb.Name), + PublicIP: &PublicIPSpec{ + Name: generatePublicIPName(c.ObjectMeta.Name), + }, + }, + } + } + + } else if lb.Type == Internal { + if lb.Name == "" { + lb.Name = generateInternalLBName(c.ObjectMeta.Name) + } + if len(lb.FrontendIPs) == 0 { + // for back compat, set the private IP to the subnet InternalLBIPAddress value. + privateIP := c.Spec.NetworkSpec.GetControlPlaneSubnet().InternalLBIPAddress + if privateIP == "" { + privateIP = DefaultInternalLBIPAddress + } + lb.FrontendIPs = []FrontendIP{ + { + Name: generateFrontendIPConfigName(lb.Name), + PrivateIPAddress: privateIP, + }, + } + } + } +} + // generateVnetName generates a virtual network name, based on the cluster name. func generateVnetName(clusterName string) string { return fmt.Sprintf("%s-%s", clusterName, "vnet") @@ -132,3 +181,23 @@ func generateNodeSecurityGroupName(clusterName string) string { func generateNodeRouteTableName(clusterName string) string { return fmt.Sprintf("%s-%s", clusterName, "node-routetable") } + +// generateInternalLBName generates a internal load balancer name, based on the cluster name. +func generateInternalLBName(clusterName string) string { + return fmt.Sprintf("%s-%s", clusterName, "internal-lb") +} + +// generatePublicLBName generates a public load balancer name, based on the cluster name. +func generatePublicLBName(clusterName string) string { + return fmt.Sprintf("%s-%s", clusterName, "public-lb") +} + +// generatePublicIPName generates a public IP name, based on the cluster name and a hash. +func generatePublicIPName(clusterName string) string { + return fmt.Sprintf("pip-%s-apiserver", clusterName) +} + +// generateFrontendIPConfigName generates a load balancer frontend IP config name. +func generateFrontendIPConfigName(lbName string) string { + return fmt.Sprintf("%s-%s", lbName, "frontEnd") +} diff --git a/api/v1alpha3/azurecluster_default_test.go b/api/v1alpha3/azurecluster_default_test.go index d8becddcb11..b4089199eef 100644 --- a/api/v1alpha3/azurecluster_default_test.go +++ b/api/v1alpha3/azurecluster_default_test.go @@ -523,3 +523,59 @@ func TestSubnetDefaults(t *testing.T) { }) } } + +func TestAPIServerLBDefaults(t *testing.T) { + cases := []struct { + name string + cluster *AzureCluster + output *AzureCluster + }{ + { + name: "no lb", + cluster: &AzureCluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "cluster-test", + }, + Spec: AzureClusterSpec{ + NetworkSpec: NetworkSpec{}, + }, + }, + output: &AzureCluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "cluster-test", + }, + Spec: AzureClusterSpec{ + NetworkSpec: NetworkSpec{ + APIServerLB: LoadBalancerSpec{ + Name: "cluster-test-public-lb", + SKU: SKUStandard, + FrontendIPs: []FrontendIP{ + { + Name: "cluster-test-public-lb-frontEnd", + PublicIP: &PublicIPSpec{ + Name: "pip-apiserver-cluster-test", + DNSName: "", + }, + }, + }, + Type: Public, + }, + }, + }, + }, + }, + } + + for _, c := range cases { + tc := c + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.cluster.setAPIServerLBDefaults() + if !reflect.DeepEqual(tc.cluster, tc.output) { + expected, _ := json.MarshalIndent(tc.output, "", "\t") + actual, _ := json.MarshalIndent(tc.cluster, "", "\t") + t.Errorf("Expected %s, got %s", string(expected), string(actual)) + } + }) + } +} diff --git a/api/v1alpha3/azurecluster_types.go b/api/v1alpha3/azurecluster_types.go index a1757ad178d..fd05f6a00b7 100644 --- a/api/v1alpha3/azurecluster_types.go +++ b/api/v1alpha3/azurecluster_types.go @@ -52,8 +52,6 @@ type AzureClusterSpec struct { // AzureClusterStatus defines the observed state of AzureCluster type AzureClusterStatus struct { - Network Network `json:"network,omitempty"` - // FailureDomains specifies the list of unique failure domains for the location/region of the cluster. // A FailureDomain maps to Availability Zone with an Azure Region (if the region support them). An // Availability Zone is a separate data center within a region and they can be used to ensure diff --git a/api/v1alpha3/azurecluster_validation.go b/api/v1alpha3/azurecluster_validation.go index 0b6ab937b81..0a8cd8f9ee9 100644 --- a/api/v1alpha3/azurecluster_validation.go +++ b/api/v1alpha3/azurecluster_validation.go @@ -18,6 +18,7 @@ package v1alpha3 import ( "fmt" + "net" "regexp" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,15 +36,15 @@ const ( // obtained from https://docs.microsoft.com/en-us/rest/api/resources/resourcegroups/createorupdate#uri-parameters resourceGroupRegex = `^[-\w\._\(\)]+$` // described in https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules - subnetRegex = `^[-\w\._]+$` - ipv4Regex = `^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$` + subnetRegex = `^[-\w\._]+$` + loadBalancerRegex = `^[-\w\._]+$` ) // validateCluster validates a cluster -func (c *AzureCluster) validateCluster() error { +func (c *AzureCluster) validateCluster(old *AzureCluster) error { var allErrs field.ErrorList allErrs = append(allErrs, c.validateClusterName()...) - allErrs = append(allErrs, c.validateClusterSpec()...) + allErrs = append(allErrs, c.validateClusterSpec(old)...) if len(allErrs) == 0 { return nil } @@ -54,9 +55,14 @@ func (c *AzureCluster) validateCluster() error { } // validateClusterSpec validates a ClusterSpec -func (c *AzureCluster) validateClusterSpec() field.ErrorList { +func (c *AzureCluster) validateClusterSpec(old *AzureCluster) field.ErrorList { + var oldNetworkSpec NetworkSpec + if old != nil { + oldNetworkSpec = old.Spec.NetworkSpec + } return validateNetworkSpec( c.Spec.NetworkSpec, + oldNetworkSpec, field.NewPath("spec").Child("networkSpec")) } @@ -79,7 +85,7 @@ func (c *AzureCluster) validateClusterName() field.ErrorList { } // validateNetworkSpec validates a NetworkSpec -func validateNetworkSpec(networkSpec NetworkSpec, fldPath *field.Path) field.ErrorList { +func validateNetworkSpec(networkSpec NetworkSpec, old NetworkSpec, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList // If the user specifies a resourceGroup for vnet, it means // that she intends to use a pre-existing vnet. In this case, @@ -91,6 +97,7 @@ func validateNetworkSpec(networkSpec NetworkSpec, fldPath *field.Path) field.Err } allErrs = append(allErrs, validateSubnets(networkSpec.Subnets, fldPath.Child("subnets"))...) } + allErrs = append(allErrs, validateAPIServerLB(networkSpec.APIServerLB, old.APIServerLB, networkSpec.GetControlPlaneSubnet().CIDRBlocks, fldPath.Child("apiServerLB"))...) if len(allErrs) == 0 { return nil } @@ -123,12 +130,6 @@ func validateSubnets(subnets Subnets, fldPath *field.Path) field.ErrorList { allErrs = append(allErrs, field.Duplicate(fldPath, subnet.Name)) } subnetNames[subnet.Name] = true - if subnet.InternalLBIPAddress != "" { - if err := validateInternalLBIPAddress(subnet.InternalLBIPAddress, - fldPath.Index(i).Child("internalLBIPAddress")); err != nil { - allErrs = append(allErrs, err) - } - } for role := range requiredSubnetRoles { if role == string(subnet.Role) { requiredSubnetRoles[role] = true @@ -151,13 +152,10 @@ func validateSubnets(subnets Subnets, fldPath *field.Path) field.ErrorList { fmt.Sprintf("required role %s not included in provided subnets", k))) } } - if len(allErrs) == 0 { - return nil - } return allErrs } -// validateSubnetName validates the Name of a Subnet +// validateSubnetName validates the Name of a Subnet. func validateSubnetName(name string, fldPath *field.Path) *field.Error { if success, _ := regexp.Match(subnetRegex, []byte(name)); !success { return field.Invalid(fldPath, name, @@ -166,15 +164,32 @@ func validateSubnetName(name string, fldPath *field.Path) *field.Error { return nil } -// validateInternalLBIPAddress validates a InternalLBIPAddress -func validateInternalLBIPAddress(address string, fldPath *field.Path) *field.Error { - if success, _ := regexp.Match(ipv4Regex, []byte(address)); !success { - return field.Invalid(fldPath, address, - fmt.Sprintf("internalLBIPAddress doesn't match regex %s", ipv4Regex)) +// validateLoadBalancerName validates the Name of a Load Balancer. +func validateLoadBalancerName(name string, fldPath *field.Path) *field.Error { + if success, _ := regexp.Match(loadBalancerRegex, []byte(name)); !success { + return field.Invalid(fldPath, name, + fmt.Sprintf("name of load balancer doesn't match regex %s", loadBalancerRegex)) } return nil } +// validateInternalLBIPAddress validates a InternalLBIPAddress. +func validateInternalLBIPAddress(address string, cidrs []string, fldPath *field.Path) *field.Error { + ip := net.ParseIP(address) + if ip == nil { + return field.Invalid(fldPath, address, + "Internal LB IP address isn't a valid IPv4 or IPv6 address") + } + for _, cidr := range cidrs { + _, subnet, _ := net.ParseCIDR(cidr) + if subnet.Contains(ip) { + return nil + } + } + return field.Invalid(fldPath, address, + fmt.Sprintf("Internal LB IP address needs to be in control plane subnet range (%s)", cidrs)) +} + // validateIngressRule validates an IngressRule func validateIngressRule(ingressRule *IngressRule, fldPath *field.Path) *field.Error { if ingressRule.Priority < 100 || ingressRule.Priority > 4096 { @@ -184,3 +199,64 @@ func validateIngressRule(ingressRule *IngressRule, fldPath *field.Path) *field.E return nil } + +func validateAPIServerLB(lb LoadBalancerSpec, old LoadBalancerSpec, cidrs []string, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + // SKU should be Standard and is immutable. + if lb.SKU != SKUStandard { + allErrs = append(allErrs, field.NotSupported(fldPath.Child("sku"), lb.SKU, []string{string(SKUStandard)})) + } + if old.SKU != "" && old.SKU != lb.SKU { + allErrs = append(allErrs, field.Invalid(fldPath.Child("sku"), lb.SKU, "API Server load balancer SKU should not be modified after AzureCluster creation.")) + } + + // Type should be Public or Internal. + if lb.Type != Internal && lb.Type != Public { + allErrs = append(allErrs, field.NotSupported(fldPath.Child("type"), lb.Type, + []string{string(Public), string(Internal)})) + } + if old.Type != "" && old.Type != lb.Type { + allErrs = append(allErrs, field.Invalid(fldPath.Child("type"), lb.Type, "API Server load balancer type should not be modified after AzureCluster creation.")) + } + + // Name should be valid. + if err := validateLoadBalancerName(lb.Name, fldPath.Child("name")); err != nil { + allErrs = append(allErrs, err) + } + if old.Name != "" && old.Name != lb.Name { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), lb.Type, "API Server load balancer name should not be modified after AzureCluster creation.")) + } + + // There should only be one IP config. + if len(lb.FrontendIPs) != 1 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("frontendIPConfigs"), lb.FrontendIPs, + fmt.Sprintf("API Server Load balancer should have 1 Frontend IP configuration"))) + } else { + // if Internal, IP config should not have a public IP. + if lb.Type == Internal { + if lb.FrontendIPs[0].PublicIP != nil { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("publicIP"), + fmt.Sprintf("Internal Load Balancers cannot have a Public IP"))) + } + if lb.FrontendIPs[0].PrivateIPAddress != "" { + if err := validateInternalLBIPAddress(lb.FrontendIPs[0].PrivateIPAddress, cidrs, + fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil { + allErrs = append(allErrs, err) + } + if len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), lb.Type, "API Server load balancer private IP should not be modified after AzureCluster creation.")) + } + } + } + + // if Public, IP config should not have a private IP. + if lb.Type == Public { + if lb.FrontendIPs[0].PrivateIPAddress != "" { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP"), + fmt.Sprintf("Public Load Balancers cannot have a Private IP"))) + } + } + } + + return allErrs +} diff --git a/api/v1alpha3/azurecluster_validation_test.go b/api/v1alpha3/azurecluster_validation_test.go index b9bcdb65c29..9b0bb1ced0b 100644 --- a/api/v1alpha3/azurecluster_validation_test.go +++ b/api/v1alpha3/azurecluster_validation_test.go @@ -110,7 +110,7 @@ func TestClusterWithPreexistingVnetValid(t *testing.T) { } t.Run(testCase.name, func(t *testing.T) { - err := testCase.cluster.validateCluster() + err := testCase.cluster.validateCluster(nil) g.Expect(err).To(BeNil()) }) } @@ -135,7 +135,7 @@ func TestClusterWithPreexistingVnetInvalid(t *testing.T) { } t.Run(testCase.name, func(t *testing.T) { - err := testCase.cluster.validateCluster() + err := testCase.cluster.validateCluster(nil) g.Expect(err).ToNot(BeNil()) }) } @@ -158,7 +158,7 @@ func TestClusterWithoutPreexistingVnetValid(t *testing.T) { testCase.cluster.Spec.NetworkSpec.Vnet.ResourceGroup = "" t.Run(testCase.name, func(t *testing.T) { - err := testCase.cluster.validateCluster() + err := testCase.cluster.validateCluster(nil) g.Expect(err).To(BeNil()) }) } @@ -177,7 +177,7 @@ func TestClusterSpecWithPreexistingVnetValid(t *testing.T) { } t.Run(testCase.name, func(t *testing.T) { - errs := testCase.cluster.validateClusterSpec() + errs := testCase.cluster.validateClusterSpec(nil) g.Expect(errs).To(BeNil()) }) } @@ -202,7 +202,7 @@ func TestClusterSpecWithPreexistingVnetInvalid(t *testing.T) { } t.Run(testCase.name, func(t *testing.T) { - errs := testCase.cluster.validateClusterSpec() + errs := testCase.cluster.validateClusterSpec(nil) g.Expect(len(errs)).To(BeNumerically(">", 0)) }) } @@ -225,7 +225,7 @@ func TestClusterSpecWithoutPreexistingVnetValid(t *testing.T) { testCase.cluster.Spec.NetworkSpec.Vnet.ResourceGroup = "" t.Run(testCase.name, func(t *testing.T) { - errs := testCase.cluster.validateClusterSpec() + errs := testCase.cluster.validateClusterSpec(nil) g.Expect(errs).To(BeNil()) }) } @@ -244,7 +244,7 @@ func TestNetworkSpecWithPreexistingVnetValid(t *testing.T) { } t.Run(testCase.name, func(t *testing.T) { - errs := validateNetworkSpec(testCase.networkSpec, field.NewPath("spec").Child("networkSpec")) + errs := validateNetworkSpec(testCase.networkSpec, NetworkSpec{}, field.NewPath("spec").Child("networkSpec")) g.Expect(errs).To(BeNil()) }) } @@ -266,7 +266,7 @@ func TestNetworkSpecWithPreexistingVnetLackRequiredSubnets(t *testing.T) { testCase.networkSpec.Subnets = testCase.networkSpec.Subnets[:1] t.Run(testCase.name, func(t *testing.T) { - errs := validateNetworkSpec(testCase.networkSpec, field.NewPath("spec").Child("networkSpec")) + errs := validateNetworkSpec(testCase.networkSpec, NetworkSpec{}, field.NewPath("spec").Child("networkSpec")) g.Expect(errs).To(HaveLen(1)) g.Expect(errs[0].Type).To(Equal(field.ErrorTypeRequired)) g.Expect(errs[0].Field).To(Equal("spec.networkSpec.subnets")) @@ -290,7 +290,7 @@ func TestNetworkSpecWithPreexistingVnetInvalidResourceGroup(t *testing.T) { testCase.networkSpec.Vnet.ResourceGroup = "invalid-name###" t.Run(testCase.name, func(t *testing.T) { - errs := validateNetworkSpec(testCase.networkSpec, field.NewPath("spec").Child("networkSpec")) + errs := validateNetworkSpec(testCase.networkSpec, NetworkSpec{}, field.NewPath("spec").Child("networkSpec")) g.Expect(errs).To(HaveLen(1)) g.Expect(errs[0].Type).To(Equal(field.ErrorTypeInvalid)) g.Expect(errs[0].Field).To(Equal("spec.networkSpec.vnet.resourceGroup")) @@ -314,7 +314,7 @@ func TestNetworkSpecWithoutPreexistingVnetValid(t *testing.T) { testCase.networkSpec.Vnet.ResourceGroup = "" t.Run(testCase.name, func(t *testing.T) { - errs := validateNetworkSpec(testCase.networkSpec, field.NewPath("spec").Child("networkSpec")) + errs := validateNetworkSpec(testCase.networkSpec, NetworkSpec{}, field.NewPath("spec").Child("networkSpec")) g.Expect(errs).To(BeNil()) }) } @@ -407,31 +407,6 @@ func TestSubnetsInvalidSubnetName(t *testing.T) { }) } -func TestSubnetsInvalidInternalLBIPAddress(t *testing.T) { - g := NewWithT(t) - - type test struct { - name string - subnets Subnets - } - - testCase := test{ - name: "subnets - invalid internal load balancer ip address", - subnets: createValidSubnets(), - } - - testCase.subnets[0].InternalLBIPAddress = "2550.1.1.1" - - t.Run(testCase.name, func(t *testing.T) { - errs := validateSubnets(testCase.subnets, - field.NewPath("spec").Child("networkSpec").Child("subnets")) - g.Expect(errs).To(HaveLen(1)) - g.Expect(errs[0].Type).To(Equal(field.ErrorTypeInvalid)) - g.Expect(errs[0].Field).To(Equal("spec.networkSpec.subnets[0].internalLBIPAddress")) - g.Expect(errs[0].BadValue).To(BeEquivalentTo("2550.1.1.1")) - }) -} - func TestSubnetsInvalidLackRequiredSubnet(t *testing.T) { g := NewWithT(t) @@ -531,15 +506,17 @@ func TestInternalLBIPAddressValid(t *testing.T) { type test struct { name string internalLBIPAddress string + cidrs []string } testCase := test{ name: "subnet name - invalid", internalLBIPAddress: "1.1.1.1", + cidrs: []string{"1.1.1.0/24"}, } t.Run(testCase.name, func(t *testing.T) { - err := validateInternalLBIPAddress(testCase.internalLBIPAddress, + err := validateInternalLBIPAddress(testCase.internalLBIPAddress, testCase.cidrs, field.NewPath("spec").Child("networkSpec").Child("subnets").Index(0).Child("internalLBIPAddress")) g.Expect(err).To(BeNil()) }) @@ -549,8 +526,9 @@ func TestInternalLBIPAddressInvalid(t *testing.T) { g := NewWithT(t) internalLBIPAddress := "1.1.1" + cidrs := []string{"1.1.1.0/24"} - err := validateInternalLBIPAddress(internalLBIPAddress, + err := validateInternalLBIPAddress(internalLBIPAddress, cidrs, field.NewPath("spec").Child("networkSpec").Child("subnets").Index(0).Child("internalLBIPAddress")) g.Expect(err).NotTo(BeNil()) g.Expect(err.Type).To(Equal(field.ErrorTypeInvalid)) @@ -595,8 +573,9 @@ func TestIngressRules(t *testing.T) { }, } for _, testCase := range tests { - + testCase := testCase t.Run(testCase.name, func(t *testing.T) { + t.Parallel() err := validateIngressRule( testCase.validRule, field.NewPath("spec").Child("networkSpec").Child("subnets").Index(0).Child("securityGroup").Child("ingressRules").Index(0), @@ -610,6 +589,206 @@ func TestIngressRules(t *testing.T) { } } +func TestValidateAPIServerLB(t *testing.T) { + g := NewWithT(t) + + testcases := []struct { + name string + lb LoadBalancerSpec + old LoadBalancerSpec + cpCIDRS []string + wantErr bool + expectedErr field.Error + }{ + { + name: "invalid SKU", + lb: LoadBalancerSpec{ + Name: "my-awesome-lb", + SKU: "Awesome", + FrontendIPs: []FrontendIP{ + { + Name: "ip-config", + }, + }, + Type: Public, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueNotSupported", + Field: "apiServerLB.sku", + BadValue: "Awesome", + Detail: "supported values: \"Standard\"", + }, + }, + { + name: "invalid Type", + lb: LoadBalancerSpec{ + Type: "Foo", + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueNotSupported", + Field: "apiServerLB.type", + BadValue: "Foo", + Detail: "supported values: \"Public\", \"Internal\"", + }, + }, + { + name: "invalid Name", + lb: LoadBalancerSpec{ + Name: "***", + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.name", + BadValue: "***", + Detail: "name of load balancer doesn't match regex ^[-\\w\\._]+$", + }, + }, + { + name: "too many IP configs", + lb: LoadBalancerSpec{ + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + }, + { + Name: "ip-2", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.frontendIPConfigs", + BadValue: []FrontendIP{ + { + Name: "ip-1", + }, + { + Name: "ip-2", + }, + }, + Detail: "API Server Load balancer should have 1 Frontend IP configuration", + }, + }, + { + name: "public LB with private IP", + lb: LoadBalancerSpec{ + Type: Public, + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + PrivateIPAddress: "10.0.0.4", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueForbidden", + Field: "apiServerLB.frontendIPConfigs[0].privateIP", + Detail: "Public Load Balancers cannot have a Private IP", + }, + }, + { + name: "internal LB with public IP", + lb: LoadBalancerSpec{ + Type: Internal, + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + PublicIP: &PublicIPSpec{ + Name: "my-invalid-ip", + }, + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueForbidden", + Field: "apiServerLB.frontendIPConfigs[0].publicIP", + Detail: "Internal Load Balancers cannot have a Public IP", + }, + }, + { + name: "internal LB with invalid private IP", + lb: LoadBalancerSpec{ + Type: Internal, + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + PrivateIPAddress: "NAIP", + }, + }, + }, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.frontendIPConfigs[0].privateIP", + BadValue: "NAIP", + Detail: "Internal LB IP address isn't a valid IPv4 or IPv6 address", + }, + }, + { + name: "internal LB with out of range private IP", + lb: LoadBalancerSpec{ + Type: Internal, + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + PrivateIPAddress: "20.1.2.3", + }, + }, + }, + cpCIDRS: []string{"10.0.0.0/24", "10.1.0.0/24"}, + wantErr: true, + expectedErr: field.Error{ + Type: "FieldValueInvalid", + Field: "apiServerLB.frontendIPConfigs[0].privateIP", + BadValue: "20.1.2.3", + Detail: "Internal LB IP address needs to be in control plane subnet range ([10.0.0.0/24 10.1.0.0/24])", + }, + }, + { + name: "internal LB with in range private IP", + lb: LoadBalancerSpec{ + Type: Internal, + SKU: SKUStandard, + Name: "my-private-lb", + FrontendIPs: []FrontendIP{ + { + Name: "ip-1", + PrivateIPAddress: "10.1.0.3", + }, + }, + }, + cpCIDRS: []string{"10.0.0.0/24", "10.1.0.0/24"}, + wantErr: false, + }, + } + + for _, test := range testcases { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + err := validateAPIServerLB(test.lb, test.old, test.cpCIDRS, field.NewPath("apiServerLB")) + if test.wantErr { + g.Expect(err).NotTo(HaveLen(0)) + found := false + for _, actual := range err { + if actual.Error() == test.expectedErr.Error() { + found = true + } + } + g.Expect(found).To(BeTrue()) + } else { + g.Expect(err).To(HaveLen(0)) + } + }) + } +} + func createValidCluster() *AzureCluster { return &AzureCluster{ ObjectMeta: metav1.ObjectMeta{ diff --git a/api/v1alpha3/azurecluster_webhook.go b/api/v1alpha3/azurecluster_webhook.go index 97e73a3c003..c6f351a4f02 100644 --- a/api/v1alpha3/azurecluster_webhook.go +++ b/api/v1alpha3/azurecluster_webhook.go @@ -50,14 +50,14 @@ func (c *AzureCluster) Default() { func (c *AzureCluster) ValidateCreate() error { clusterlog.Info("validate create", "name", c.Name) - return c.validateCluster() + return c.validateCluster(nil) } // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type -func (c *AzureCluster) ValidateUpdate(old runtime.Object) error { +func (c *AzureCluster) ValidateUpdate(oldRaw runtime.Object) error { clusterlog.Info("validate update", "name", c.Name) - - return c.validateCluster() + old := oldRaw.(*AzureCluster) + return c.validateCluster(old) } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type diff --git a/api/v1alpha3/tags.go b/api/v1alpha3/tags.go index 318966a8cbc..0c8587b567d 100644 --- a/api/v1alpha3/tags.go +++ b/api/v1alpha3/tags.go @@ -107,9 +107,6 @@ const ( // APIServerRole describes the value for the apiserver role APIServerRole = "apiserver" - // InternalRole describes the value for the internal role - InternalRole = "internal" - // NodeOutboundRole describes the value for the node outbound LB role NodeOutboundRole = "nodeOutbound" diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index aa14dd76675..2c59f0c23b9 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -27,15 +27,6 @@ const ( Node string = "node" ) -// Network encapsulates the state of Azure networking resources. -type Network struct { - // APIServerLB is the Kubernetes API server load balancer. - APIServerLB LoadBalancer `json:"apiServerLb,omitempty"` - - // APIServerIP is the Kubernetes API server public IP address. - APIServerIP PublicIP `json:"apiServerIp,omitempty"` -} - // NetworkSpec specifies what the Azure networking resources should look like. type NetworkSpec struct { // Vnet is the configuration for the Azure virtual network. @@ -45,6 +36,10 @@ type NetworkSpec struct { // Subnets is the configuration for the control-plane subnet and the node subnet. // +optional Subnets Subnets `json:"subnets,omitempty"` + + // APIServerLB is the configuration for the control-plane load balancer. + // +optional + APIServerLB LoadBalancerSpec `json:"apiServerLB,omitempty"` } // VnetSpec configures an Azure virtual network. @@ -145,42 +140,48 @@ type IngressRule struct { // IngressRules is a slice of Azure ingress rules for security groups. type IngressRules []*IngressRule -// PublicIP defines an Azure public IP address. -type PublicIP struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - IPAddress string `json:"ipAddress,omitempty"` - DNSName string `json:"dnsName,omitempty"` +// LoadBalancerSpec defines an Azure load balancer. +type LoadBalancerSpec struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + SKU SKU `json:"sku,omitempty"` + FrontendIPs []FrontendIP `json:"frontendIPs,omitempty"` + Type LBType `json:"type,omitempty"` } -// LoadBalancer defines an Azure load balancer. -type LoadBalancer struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - SKU SKU `json:"sku,omitempty"` - FrontendIPConfig FrontendIPConfig `json:"frontendIpConfig,omitempty"` - BackendPool BackendPool `json:"backendPool,omitempty"` - Tags Tags `json:"tags,omitempty"` -} - -// FrontendIPConfig - DO NOT USE -// this empty struct is here to preserve backwards compatibility and should be removed in v1alpha4 -type FrontendIPConfig struct{} - // SKU defines an Azure load balancer SKU. type SKU string const ( - // SKUBasic is the value for the Azure load balancer Basic SKU - SKUBasic = SKU("Basic") - // SKUStandard is the value for the Azure load balancer Standard SKU + // SKUStandard is the value for the Azure load balancer Standard SKU. SKUStandard = SKU("Standard") ) -// BackendPool defines a load balancer backend pool. -type BackendPool struct { - Name string `json:"name,omitempty"` - ID string `json:"id,omitempty"` +// LBType defines an Azure load balancer Type. +type LBType string + +const ( + // Internal is the value for the Azure load b alancer internal type. + Internal = LBType("Internal") + // Public is the value for the Azure load balancer public type. + Public = LBType("Public") +) + +// FrontendIP defines a load balancer frontend IP configuration. +type FrontendIP struct { + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + // +optional + PrivateIPAddress string `json:"privateIP,omitempty"` + // +optional + PublicIP *PublicIPSpec `json:"publicIP,omitempty"` +} + +// PublicIPSpec defines the inputs to create an Azure public IP address. +type PublicIPSpec struct { + Name string `json:"name"` + // +optional + DNSName string `json:"dnsName,omitempty"` } // VMState describes the state of an Azure virtual machine. @@ -396,6 +397,7 @@ type SubnetSpec struct { // InternalLBIPAddress is the IP address that will be used as the internal LB private IP. // For the control plane subnet only. // +optional + // Deprecated: Use LoadBalancer private IP instead InternalLBIPAddress string `json:"internalLBIPAddress,omitempty"` // SecurityGroup defines the NSG (network security group) that should be attached to this subnet. diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index f17243cf6ea..3169159ce1b 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -138,7 +138,6 @@ func (in *AzureClusterSpec) DeepCopy() *AzureClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzureClusterStatus) DeepCopyInto(out *AzureClusterStatus) { *out = *in - in.Network.DeepCopyInto(&out.Network) if in.FailureDomains != nil { in, out := &in.FailureDomains, &out.FailureDomains *out = make(apiv1alpha3.FailureDomains, len(*in)) @@ -452,21 +451,6 @@ func (in *AzureSharedGalleryImage) DeepCopy() *AzureSharedGalleryImage { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BackendPool) DeepCopyInto(out *BackendPool) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackendPool. -func (in *BackendPool) DeepCopy() *BackendPool { - if in == nil { - return nil - } - out := new(BackendPool) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BuildParams) DeepCopyInto(out *BuildParams) { *out = *in @@ -550,16 +534,21 @@ func (in *DiskEncryptionSetParameters) DeepCopy() *DiskEncryptionSetParameters { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FrontendIPConfig) DeepCopyInto(out *FrontendIPConfig) { +func (in *FrontendIP) DeepCopyInto(out *FrontendIP) { *out = *in + if in.PublicIP != nil { + in, out := &in.PublicIP, &out.PublicIP + *out = new(PublicIPSpec) + **out = **in + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FrontendIPConfig. -func (in *FrontendIPConfig) DeepCopy() *FrontendIPConfig { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FrontendIP. +func (in *FrontendIP) DeepCopy() *FrontendIP { if in == nil { return nil } - out := new(FrontendIPConfig) + out := new(FrontendIP) in.DeepCopyInto(out) return out } @@ -655,25 +644,23 @@ func (in IngressRules) DeepCopy() IngressRules { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) { +func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = *in - out.FrontendIPConfig = in.FrontendIPConfig - out.BackendPool = in.BackendPool - if in.Tags != nil { - in, out := &in.Tags, &out.Tags - *out = make(Tags, len(*in)) - for key, val := range *in { - (*out)[key] = val + if in.FrontendIPs != nil { + in, out := &in.FrontendIPs, &out.FrontendIPs + *out = make([]FrontendIP, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer. -func (in *LoadBalancer) DeepCopy() *LoadBalancer { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancerSpec. +func (in *LoadBalancerSpec) DeepCopy() *LoadBalancerSpec { if in == nil { return nil } - out := new(LoadBalancer) + out := new(LoadBalancerSpec) in.DeepCopyInto(out) return out } @@ -698,23 +685,6 @@ func (in *ManagedDisk) DeepCopy() *ManagedDisk { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Network) DeepCopyInto(out *Network) { - *out = *in - in.APIServerLB.DeepCopyInto(&out.APIServerLB) - out.APIServerIP = in.APIServerIP -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. -func (in *Network) DeepCopy() *Network { - if in == nil { - return nil - } - out := new(Network) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { *out = *in @@ -730,6 +700,7 @@ func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { } } } + in.APIServerLB.DeepCopyInto(&out.APIServerLB) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkSpec. @@ -764,16 +735,16 @@ func (in *OSDisk) DeepCopy() *OSDisk { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PublicIP) DeepCopyInto(out *PublicIP) { +func (in *PublicIPSpec) DeepCopyInto(out *PublicIPSpec) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublicIP. -func (in *PublicIP) DeepCopy() *PublicIP { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublicIPSpec. +func (in *PublicIPSpec) DeepCopy() *PublicIPSpec { if in == nil { return nil } - out := new(PublicIP) + out := new(PublicIPSpec) in.DeepCopyInto(out) return out } diff --git a/cloud/defaults.go b/cloud/defaults.go index e33fec70b37..b02caf47dcc 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -28,13 +28,6 @@ import ( const ( // DefaultUserName is the default username for created vm DefaultUserName = "capi" - // DefaultInternalLBIPAddress is the default internal load balancer ip address - DefaultInternalLBIPAddress = "10.0.0.100" -) - -const ( - // DefaultInternalLBIPv6Address is the default internal load balancer ip address - DefaultInternalLBIPv6Address = "2001:1234:5678:9abc::100" ) const ( @@ -46,23 +39,13 @@ const ( LatestVersion = "latest" ) -// GenerateInternalLBName generates a internal load balancer name, based on the cluster name. -func GenerateInternalLBName(clusterName string) string { - return fmt.Sprintf("%s-%s", clusterName, "internal-lb") -} - -// GeneratePublicLBName generates a public load balancer name, based on the cluster name. -func GeneratePublicLBName(clusterName string) string { - return fmt.Sprintf("%s-%s", clusterName, "public-lb") -} - // GenerateBackendAddressPoolName generates a load balancer backend address pool name. func GenerateBackendAddressPoolName(lbName string) string { return fmt.Sprintf("%s-%s", lbName, "backendPool") } -// GenerateOutboundBackendddressPoolName generates a load balancer outbound backend address pool name. -func GenerateOutboundBackendddressPoolName(lbName string) string { +// GenerateOutboundBackendAddressPoolName generates a load balancer outbound backend address pool name. +func GenerateOutboundBackendAddressPoolName(lbName string) string { return fmt.Sprintf("%s-%s", lbName, "outboundBackendPool") } @@ -71,11 +54,6 @@ func GenerateFrontendIPConfigName(lbName string) string { return fmt.Sprintf("%s-%s", lbName, "frontEnd") } -// GeneratePublicIPName generates a public IP name, based on the cluster name and a hash. -func GeneratePublicIPName(clusterName, hash string) string { - return fmt.Sprintf("%s-%s", clusterName, hash) -} - // GenerateNodeOutboundIPName generates a public IP name, based on the cluster name. func GenerateNodeOutboundIPName(clusterName string) string { return fmt.Sprintf("pip-%s-node-outbound", clusterName) diff --git a/cloud/interfaces.go b/cloud/interfaces.go index 6937d908bd3..f925ca0dd13 100644 --- a/cloud/interfaces.go +++ b/cloud/interfaces.go @@ -65,6 +65,8 @@ type NetworkDescriber interface { IsIPv6Enabled() bool NodeRouteTable() *infrav1.RouteTable ControlPlaneRouteTable() *infrav1.RouteTable + APIServerLBName() string + NodeOutboundLBName() string } // ClusterDescriber is an interface which can get common Azure Cluster information. diff --git a/cloud/mocks/service_mock.go b/cloud/mocks/service_mock.go index 7f2ca2ba4a3..1154997cca5 100644 --- a/cloud/mocks/service_mock.go +++ b/cloud/mocks/service_mock.go @@ -438,6 +438,34 @@ func (mr *MockNetworkDescriberMockRecorder) ControlPlaneRouteTable() *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockNetworkDescriber)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockNetworkDescriber) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockNetworkDescriberMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockNetworkDescriber)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockNetworkDescriber) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockNetworkDescriberMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockNetworkDescriber)(nil).NodeOutboundLBName)) +} + // MockClusterDescriber is a mock of ClusterDescriber interface. type MockClusterDescriber struct { ctrl *gomock.Controller @@ -889,3 +917,31 @@ func (mr *MockClusterScoperMockRecorder) ControlPlaneRouteTable() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockClusterScoper)(nil).ControlPlaneRouteTable)) } + +// APIServerLBName mocks base method. +func (m *MockClusterScoper) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockClusterScoperMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockClusterScoper)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockClusterScoper) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockClusterScoperMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockClusterScoper)(nil).NodeOutboundLBName)) +} diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index d5e3e14d5c4..19137966622 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -19,7 +19,9 @@ package scope import ( "context" "fmt" + "hash/fnv" "strconv" + "strings" "k8s.io/utils/net" @@ -100,57 +102,53 @@ func (s *ClusterScope) Authorizer() autorest.Authorizer { return s.AzureClients.Authorizer } -// Network returns the cluster network object. -func (s *ClusterScope) Network() *infrav1.Network { - return &s.AzureCluster.Status.Network -} - // PublicIPSpecs returns the public IP specs. func (s *ClusterScope) PublicIPSpecs() []azure.PublicIPSpec { - return []azure.PublicIPSpec{ + specs := []azure.PublicIPSpec{ { - Name: azure.GenerateNodeOutboundIPName(s.ClusterName()), + Name: azure.GenerateNodeOutboundIPName(s.NodeOutboundLBName()), }, - { - Name: s.Network().APIServerIP.Name, - DNSName: s.Network().APIServerIP.DNSName, + } + if s.APIServerLB().Type == infrav1.Public { + specs = append(specs, azure.PublicIPSpec{ + Name: s.APIServerPublicIP().Name, + DNSName: s.APIServerPublicIP().DNSName, IsIPv6: false, // currently azure requires a ipv4 lb rule to enable ipv6 - }, + }) } + return specs } // LBSpecs returns the load balancer specs. func (s *ClusterScope) LBSpecs() []azure.LBSpec { - specs := []azure.LBSpec{ + return []azure.LBSpec{ { - // Public API Server LB - Name: azure.GeneratePublicLBName(s.ClusterName()), - PublicIPName: s.Network().APIServerIP.Name, - APIServerPort: s.APIServerPort(), - Role: infrav1.APIServerRole, + // Control Plane LB + Name: s.APIServerLB().Name, + SubnetName: s.ControlPlaneSubnet().Name, + SubnetCidrs: s.ControlPlaneSubnet().CIDRBlocks, + FrontendIPConfigs: s.APIServerLB().FrontendIPs, + APIServerPort: s.APIServerPort(), + Type: s.APIServerLB().Type, + Role: infrav1.APIServerRole, + BackendPoolName: azure.GenerateBackendAddressPoolName(s.APIServerLB().Name), }, { // Public Node outbound LB - Name: s.ClusterName(), - PublicIPName: azure.GenerateNodeOutboundIPName(s.ClusterName()), - Role: infrav1.NodeOutboundRole, + Name: s.NodeOutboundLBName(), + FrontendIPConfigs: []infrav1.FrontendIP{ + { + Name: azure.GenerateFrontendIPConfigName(s.NodeOutboundLBName()), + PublicIP: &infrav1.PublicIPSpec{ + Name: azure.GenerateNodeOutboundIPName(s.NodeOutboundLBName()), + }, + }, + }, + Type: infrav1.Public, + BackendPoolName: azure.GenerateOutboundBackendAddressPoolName(s.NodeOutboundLBName()), + Role: infrav1.NodeOutboundRole, }, } - if !s.IsIPv6Enabled() { - // for now no internal LB is created for ipv6 enabled clusters - // getAvailablePrivateIP() does not work with IPv6 - specs = append(specs, azure.LBSpec{ - // Internal control plane LB - Name: azure.GenerateInternalLBName(s.ClusterName()), - SubnetName: s.ControlPlaneSubnet().Name, - SubnetCidrs: s.ControlPlaneSubnet().CIDRBlocks, - PrivateIPAddress: s.ControlPlaneSubnet().InternalLBIPAddress, - APIServerPort: s.APIServerPort(), - Role: infrav1.InternalRole, - }) - } - - return specs } // RouteTableSpecs returns the node route table @@ -183,13 +181,12 @@ func (s *ClusterScope) NSGSpecs() []azure.NSGSpec { func (s *ClusterScope) SubnetSpecs() []azure.SubnetSpec { return []azure.SubnetSpec{ { - Name: s.ControlPlaneSubnet().Name, - CIDRs: s.ControlPlaneSubnet().CIDRBlocks, - VNetName: s.Vnet().Name, - SecurityGroupName: s.ControlPlaneSubnet().SecurityGroup.Name, - Role: s.ControlPlaneSubnet().Role, - RouteTableName: s.ControlPlaneSubnet().RouteTable.Name, - InternalLBIPAddress: s.ControlPlaneSubnet().InternalLBIPAddress, + Name: s.ControlPlaneSubnet().Name, + CIDRs: s.ControlPlaneSubnet().CIDRBlocks, + VNetName: s.Vnet().Name, + SecurityGroupName: s.ControlPlaneSubnet().SecurityGroup.Name, + Role: s.ControlPlaneSubnet().Role, + RouteTableName: s.ControlPlaneSubnet().RouteTable.Name, }, { Name: s.NodeSubnet().Name, @@ -258,6 +255,31 @@ func (s *ClusterScope) NodeRouteTable() *infrav1.RouteTable { return &s.AzureCluster.Spec.NetworkSpec.GetNodeSubnet().RouteTable } +// APIServerLB returns the cluster API Server load balancer. +func (s *ClusterScope) APIServerLB() *infrav1.LoadBalancerSpec { + return &s.AzureCluster.Spec.NetworkSpec.APIServerLB +} + +// APIServerLBName returns the API Server LB name. +func (s *ClusterScope) APIServerLBName() string { + return s.APIServerLB().Name +} + +// APIServerPublicIP returns the API Server public IP. +func (s *ClusterScope) APIServerPublicIP() *infrav1.PublicIPSpec { + return s.APIServerLB().FrontendIPs[0].PublicIP +} + +// APIServerPrivateIP returns the API Server private IP. +func (s *ClusterScope) APIServerPrivateIP() string { + return s.APIServerLB().FrontendIPs[0].PrivateIPAddress +} + +// NodeOutboundLBName returns the name of the node outbound LB. +func (s *ClusterScope) NodeOutboundLBName() string { + return s.ClusterName() +} + // ResourceGroup returns the cluster resource group. func (s *ClusterScope) ResourceGroup() string { return s.AzureCluster.Spec.ResourceGroup @@ -278,9 +300,26 @@ func (s *ClusterScope) Location() string { return s.AzureCluster.Spec.Location } -// GenerateFQDN generates a fully qualified domain name, based on the public IP name and cluster location. -func (s *ClusterScope) GenerateFQDN() string { - return fmt.Sprintf("%s.%s.%s", s.Network().APIServerIP.Name, s.Location(), s.AzureClients.ResourceManagerVMDNSSuffix) +// GenerateFQDN generates a fully qualified domain name, based on a hash, cluster name and cluster location. +func (s *ClusterScope) GenerateFQDN(ipName string) string { + h := fnv.New32a() + if _, err := h.Write([]byte(fmt.Sprintf("%s/%s/%s", s.SubscriptionID(), s.ResourceGroup(), ipName))); err != nil { + return "" + } + hash := fmt.Sprintf("%x", h.Sum32()) + return strings.ToLower(fmt.Sprintf("%s-%s.%s.%s", s.ClusterName(), hash, s.Location(), s.AzureClients.ResourceManagerVMDNSSuffix)) +} + +// GenerateLegacyFQDN generates an IP name and a fully qualified domain name, based on a hash, cluster name and cluster location. +// DEPRECATED: use GenerateFQDN instead. +func (s *ClusterScope) GenerateLegacyFQDN() (string, string) { + h := fnv.New32a() + if _, err := h.Write([]byte(fmt.Sprintf("%s/%s/%s", s.SubscriptionID(), s.ResourceGroup(), s.ClusterName()))); err != nil { + return "", "" + } + ipName := fmt.Sprintf("%s-%x", s.ClusterName(), h.Sum32()) + fqdn := fmt.Sprintf("%s.%s.%s", ipName, s.Location(), s.AzureClients.ResourceManagerVMDNSSuffix) + return ipName, fqdn } // ListOptionsLabelSelector returns a ListOptions with a label selector for clusterName. @@ -332,6 +371,14 @@ func (s *ClusterScope) APIServerPort() int32 { return 6443 } +// APIServerHost returns the APIServerHost used to reach the API server. +func (s *ClusterScope) APIServerHost() string { + if s.APIServerLB().Type == infrav1.Internal { + return s.APIServerPrivateIP() + } + return s.APIServerPublicIP().DNSName +} + // SetFailureDomain will set the spec for a for a given key func (s *ClusterScope) SetFailureDomain(id string, spec clusterv1.FailureDomainSpec) { if s.AzureCluster.Status.FailureDomains == nil { @@ -367,3 +414,34 @@ func (s *ClusterScope) SetControlPlaneIngressRules() { } } } + +// SetDNSName sets the API Server public IP DNS name. +func (s *ClusterScope) SetDNSName() { + // for back compat, set the old API Server defaults if no API Server Spec has been set by new webhooks. + lb := s.APIServerLB() + if lb == nil || lb.Name == "" { + lbName := fmt.Sprintf("%s-%s", s.ClusterName(), "public-lb") + ip, dns := s.GenerateLegacyFQDN() + lb = &infrav1.LoadBalancerSpec{ + Name: lbName, + SKU: infrav1.SKUStandard, + FrontendIPs: []infrav1.FrontendIP{ + { + Name: azure.GenerateFrontendIPConfigName(lbName), + PublicIP: &infrav1.PublicIPSpec{ + Name: ip, + DNSName: dns, + }, + }, + }, + Type: infrav1.Public, + } + lb.DeepCopyInto(s.APIServerLB()) + } + // Generate valid FQDN if not set. + if s.APIServerLB().Type == infrav1.Public { + if s.APIServerPublicIP().DNSName == "" { + s.APIServerPublicIP().DNSName = s.GenerateFQDN(s.APIServerPublicIP().Name) + } + } +} diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index dedbdbe7580..0a9625adbf2 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -136,7 +136,7 @@ func (m *MachineScope) InboundNatSpecs() []azure.InboundNatSpec { return []azure.InboundNatSpec{ { Name: m.Name(), - LoadBalancerName: azure.GeneratePublicLBName(m.ClusterName()), + LoadBalancerName: m.APIServerLBName(), }, } } @@ -157,19 +157,12 @@ func (m *MachineScope) NICSpecs() []azure.NICSpec { EnableIPForwarding: m.AzureMachine.Spec.EnableIPForwarding, } if m.Role() == infrav1.ControlPlane { - publicLBName := azure.GeneratePublicLBName(m.ClusterName()) - spec.PublicLBName = publicLBName - spec.PublicLBAddressPoolName = azure.GenerateBackendAddressPoolName(publicLBName) - spec.PublicLBNATRuleName = m.Name() - if !m.IsIPv6Enabled() { - internalLBName := azure.GenerateInternalLBName(m.ClusterName()) - spec.InternalLBName = internalLBName - spec.InternalLBAddressPoolName = azure.GenerateBackendAddressPoolName(internalLBName) - } + spec.LBName = m.APIServerLBName() + spec.LBBackendAddressPoolName = azure.GenerateBackendAddressPoolName(m.APIServerLBName()) + spec.LBNATRuleName = m.Name() } else if m.Role() == infrav1.Node { - publicLBName := m.ClusterName() - spec.PublicLBName = publicLBName - spec.PublicLBAddressPoolName = azure.GenerateOutboundBackendddressPoolName(publicLBName) + spec.LBName = m.NodeOutboundLBName() + spec.LBBackendAddressPoolName = azure.GenerateOutboundBackendAddressPoolName(m.NodeOutboundLBName()) } specs := []azure.NICSpec{spec} if m.AzureMachine.Spec.AllocatePublicIP == true { diff --git a/cloud/scope/machinepool.go b/cloud/scope/machinepool.go index 6cabcc75abd..144685744d3 100644 --- a/cloud/scope/machinepool.go +++ b/cloud/scope/machinepool.go @@ -112,8 +112,8 @@ func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec { SubnetName: m.NodeSubnet().Name, VNetName: m.Vnet().Name, VNetResourceGroup: m.Vnet().ResourceGroup, - PublicLBName: m.ClusterName(), - PublicLBAddressPoolName: azure.GenerateOutboundBackendddressPoolName(m.ClusterName()), + PublicLBName: m.NodeOutboundLBName(), + PublicLBAddressPoolName: azure.GenerateOutboundBackendAddressPoolName(m.NodeOutboundLBName()), AcceleratedNetworking: m.AzureMachinePool.Spec.Template.AcceleratedNetworking, Identity: m.AzureMachinePool.Spec.Identity, UserAssignedIdentities: m.AzureMachinePool.Spec.UserAssignedIdentities, diff --git a/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go b/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go index bca86fd08cc..fb21e534258 100644 --- a/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go +++ b/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go @@ -398,6 +398,34 @@ func (mr *MockBastionScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockBastionScope)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockBastionScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockBastionScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockBastionScope)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockBastionScope) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockBastionScopeMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockBastionScope)(nil).NodeOutboundLBName)) +} + // BastionSpecs mocks base method. func (m *MockBastionScope) BastionSpecs() []azure.BastionSpec { m.ctrl.T.Helper() diff --git a/cloud/services/loadbalancers/loadbalancers.go b/cloud/services/loadbalancers/loadbalancers.go index 1bf8cad2112..5c8e505f1f8 100644 --- a/cloud/services/loadbalancers/loadbalancers.go +++ b/cloud/services/loadbalancers/loadbalancers.go @@ -18,7 +18,6 @@ package loadbalancers import ( "context" - "strings" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest/to" @@ -36,50 +35,15 @@ func (s *Service) Reconcile(ctx context.Context) error { defer span.End() for _, lbSpec := range s.Scope.LBSpecs() { - frontEndIPConfigName := azure.GenerateFrontendIPConfigName(lbSpec.Name) - backEndAddressPoolName := azure.GenerateBackendAddressPoolName(lbSpec.Name) - if lbSpec.Role == infrav1.NodeOutboundRole { - backEndAddressPoolName = azure.GenerateOutboundBackendddressPoolName(lbSpec.Name) - } - s.Scope.V(2).Info("creating load balancer", "load balancer", lbSpec.Name) - var frontIPConfig network.FrontendIPConfigurationPropertiesFormat - if lbSpec.Role == infrav1.InternalRole { - var privateIP string - internalLB, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), lbSpec.Name) - if err == nil { - ipConfigs := internalLB.LoadBalancerPropertiesFormat.FrontendIPConfigurations - if ipConfigs != nil && len(*ipConfigs) > 0 { - privateIP = to.String((*ipConfigs)[0].FrontendIPConfigurationPropertiesFormat.PrivateIPAddress) - } - } else if azure.ResourceNotFound(err) { - s.Scope.V(2).Info("internalLB not found in RG", "internal lb", lbSpec.Name, "resource group", s.Scope.ResourceGroup()) - privateIP, err = s.getAvailablePrivateIP(ctx, s.Scope.Vnet().ResourceGroup, s.Scope.Vnet().Name, lbSpec.PrivateIPAddress, lbSpec.SubnetCidrs) - if err != nil { - return err - } - s.Scope.V(2).Info("setting internal load balancer IP", "private ip", privateIP) - } else { - return errors.Wrap(err, "failed to look for existing internal LB") - } - frontIPConfig = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Static, - Subnet: &network.Subnet{ - ID: to.StringPtr(azure.SubnetID(s.Scope.SubscriptionID(), s.Scope.Vnet().ResourceGroup, s.Scope.Vnet().Name, lbSpec.SubnetName)), - }, - PrivateIPAddress: to.StringPtr(privateIP), - } - } else { - frontIPConfig = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Dynamic, - PublicIPAddress: &network.PublicIPAddress{ - ID: to.StringPtr(azure.PublicIPID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.PublicIPName)), - }, - } + frontendIPConfigs, frontendIDs, err := s.getFrontendIPConfigs(lbSpec) + if err != nil { + return err } lb := network.LoadBalancer{ + Name: to.StringPtr(lbSpec.Name), Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ @@ -89,30 +53,21 @@ func (s *Service) Reconcile(ctx context.Context) error { Additional: s.Scope.AdditionalTags(), })), LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ - { - Name: &frontEndIPConfigName, - FrontendIPConfigurationPropertiesFormat: &frontIPConfig, - }, - }, + FrontendIPConfigurations: &frontendIPConfigs, BackendAddressPools: &[]network.BackendAddressPool{ { - Name: &backEndAddressPoolName, + Name: to.StringPtr(lbSpec.BackendPoolName), }, }, OutboundRules: &[]network.OutboundRule{ { Name: to.StringPtr("OutboundNATAllProtocols"), OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), - FrontendIPConfigurations: &[]network.SubResource{ - { - ID: to.StringPtr(azure.FrontendIPConfigID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, frontEndIPConfigName)), - }, - }, + Protocol: network.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(4), + FrontendIPConfigurations: &frontendIDs, BackendAddressPool: &network.SubResource{ - ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, backEndAddressPoolName)), + ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)), }, }, }, @@ -120,7 +75,7 @@ func (s *Service) Reconcile(ctx context.Context) error { }, } - if lbSpec.Role == infrav1.APIServerRole || lbSpec.Role == infrav1.InternalRole { + if lbSpec.Role == infrav1.APIServerRole { probeName := "HTTPSProbe" lb.LoadBalancerPropertiesFormat.Probes = &[]network.Probe{ { @@ -134,41 +89,42 @@ func (s *Service) Reconcile(ctx context.Context) error { }, }, } - lbRule := network.LoadBalancingRule{ - Name: to.StringPtr("LBRuleHTTPS"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - Protocol: network.TransportProtocolTCP, - FrontendPort: to.Int32Ptr(lbSpec.APIServerPort), - BackendPort: to.Int32Ptr(lbSpec.APIServerPort), - IdleTimeoutInMinutes: to.Int32Ptr(4), - EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, - FrontendIPConfiguration: &network.SubResource{ - ID: to.StringPtr(azure.FrontendIPConfigID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, frontEndIPConfigName)), - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, backEndAddressPoolName)), - }, - Probe: &network.SubResource{ - ID: to.StringPtr(azure.ProbeID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, probeName)), + // We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule. + // For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections. + var frontendIPConfig network.SubResource + if len(frontendIDs) != 0 { + frontendIPConfig = frontendIDs[0] + } + lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &[]network.LoadBalancingRule{ + { + Name: to.StringPtr("LBRuleHTTPS"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + DisableOutboundSnat: to.BoolPtr(true), + Protocol: network.TransportProtocolTCP, + FrontendPort: to.Int32Ptr(lbSpec.APIServerPort), + BackendPort: to.Int32Ptr(lbSpec.APIServerPort), + IdleTimeoutInMinutes: to.Int32Ptr(4), + EnableFloatingIP: to.BoolPtr(false), + LoadDistribution: network.LoadDistributionDefault, + FrontendIPConfiguration: &frontendIPConfig, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, lbSpec.BackendPoolName)), + }, + Probe: &network.SubResource{ + ID: to.StringPtr(azure.ProbeID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, probeName)), + }, }, }, } - - if lbSpec.Role == infrav1.APIServerRole { - // We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule. - // For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections. - lbRule.LoadBalancingRulePropertiesFormat.DisableOutboundSnat = to.BoolPtr(true) - } else if lbSpec.Role == infrav1.InternalRole { + if lbSpec.Type == infrav1.Internal { lb.LoadBalancerPropertiesFormat.OutboundRules = nil } - lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &[]network.LoadBalancingRule{lbRule} } - err := s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbSpec.Name, lb) + err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbSpec.Name, lb) if err != nil { - return errors.Wrapf(err, "failed to create load balancer %s", lbSpec.Name) + return errors.Wrapf(err, "failed to create load balancer \"%s\"", lbSpec.Name) } s.Scope.V(2).Info("successfully created load balancer", "load balancer", lbSpec.Name) @@ -197,37 +153,36 @@ func (s *Service) Delete(ctx context.Context) error { return nil } -// getAvailablePrivateIP checks if the desired private IP address is available in a virtual network. -// If the IP address is taken or empty, it will make an attempt to find an available IP in the same subnet -// NOTE: this does not work for VNets with ipv6 CIDRs currently -func (s *Service) getAvailablePrivateIP(ctx context.Context, resourceGroup, vnetName, PreferredIPAddress string, subnetCIDRs []string) (string, error) { - ctx, span := tele.Tracer().Start(ctx, "loadbalancers.Service.getAvailablePrivateIP") +func (s *Service) getFrontendIPConfigs(lbSpec azure.LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource, error) { + ctx, span := tele.Tracer().Start(ctx, "loadbalancers.Service.getFrontendIPConfigs") defer span.End() - if len(subnetCIDRs) == 0 { - return "", errors.Errorf("failed to find available IP: control plane subnet CIDRs should not be empty") - } - ip := PreferredIPAddress - if ip == "" { - ip = azure.DefaultInternalLBIPAddress - subnetCIDR := subnetCIDRs[0] - if subnetCIDR != infrav1.DefaultControlPlaneSubnetCIDR { - // If the user provided a custom subnet CIDR without providing a private IP, try finding an available IP in the subnet space - index := strings.LastIndex(subnetCIDR, ".") - ip = subnetCIDR[0:(index+1)] + "0" - } - } - - result, err := s.VirtualNetworksClient.CheckIPAddressAvailability(ctx, resourceGroup, vnetName, ip) - if err != nil { - return "", errors.Wrap(err, "failed to check IP availability") - } - if !to.Bool(result.Available) { - if len(to.StringSlice(result.AvailableIPAddresses)) == 0 { - return "", errors.Errorf("IP %s is not available in VNet %s and there were no other available IPs found", ip, vnetName) + var frontendIPConfigurations []network.FrontendIPConfiguration + frontendIDs := make([]network.SubResource, 0) + for _, ipConfig := range lbSpec.FrontendIPConfigs { + var properties network.FrontendIPConfigurationPropertiesFormat + if lbSpec.Type == infrav1.Internal { + properties = network.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.Static, + Subnet: &network.Subnet{ + ID: to.StringPtr(azure.SubnetID(s.Scope.SubscriptionID(), s.Scope.Vnet().ResourceGroup, s.Scope.Vnet().Name, lbSpec.SubnetName)), + }, + PrivateIPAddress: to.StringPtr(ipConfig.PrivateIPAddress), + } + } else { + properties = network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ + ID: to.StringPtr(azure.PublicIPID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), ipConfig.PublicIP.Name)), + }, + } } - // TODO: make sure that the returned IP is in the right subnet since this check is done at the VNet level - ip = to.StringSlice(result.AvailableIPAddresses)[0] + frontendIPConfigurations = append(frontendIPConfigurations, network.FrontendIPConfiguration{ + FrontendIPConfigurationPropertiesFormat: &properties, + Name: to.StringPtr(ipConfig.Name), + }) + frontendIDs = append(frontendIDs, network.SubResource{ + ID: to.StringPtr(azure.FrontendIPConfigID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), lbSpec.Name, ipConfig.Name)), + }) } - return ip, nil + return frontendIPConfigurations, frontendIDs, nil } diff --git a/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go b/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go index c0657266def..d8029e1bda3 100644 --- a/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go +++ b/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go @@ -398,6 +398,34 @@ func (mr *MockLBScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockLBScope)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockLBScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockLBScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockLBScope)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockLBScope) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockLBScopeMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockLBScope)(nil).NodeOutboundLBName)) +} + // LBSpecs mocks base method. func (m *MockLBScope) LBSpecs() []azure.LBSpec { m.ctrl.T.Helper() diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index f4b99f713b4..ac0d893e765 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -56,30 +56,22 @@ func (s *Service) Reconcile(ctx context.Context) error { nicConfig.PrivateIPAddress = to.StringPtr(nicSpec.StaticIPAddress) } - backendAddressPools := []network.BackendAddressPool{} - if nicSpec.PublicLBName != "" { - if nicSpec.PublicLBAddressPoolName != "" { - backendAddressPools = append(backendAddressPools, - network.BackendAddressPool{ - ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.PublicLBName, nicSpec.PublicLBAddressPoolName)), - }) + if nicSpec.LBName != "" { + if nicSpec.LBBackendAddressPoolName != "" { + nicConfig.LoadBalancerBackendAddressPools = &[]network.BackendAddressPool{ + { + ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.LBName, nicSpec.LBBackendAddressPoolName)), + }, + } } - if nicSpec.PublicLBNATRuleName != "" { + if nicSpec.LBNATRuleName != "" { nicConfig.LoadBalancerInboundNatRules = &[]network.InboundNatRule{ { - ID: to.StringPtr(azure.NATRuleID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.PublicLBName, nicSpec.PublicLBNATRuleName)), + ID: to.StringPtr(azure.NATRuleID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.LBName, nicSpec.LBNATRuleName)), }, } } } - if nicSpec.InternalLBName != "" && nicSpec.InternalLBAddressPoolName != "" { - // only control planes have an attached internal LB - backendAddressPools = append(backendAddressPools, - network.BackendAddressPool{ - ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.InternalLBName, nicSpec.InternalLBAddressPoolName)), - }) - } - nicConfig.LoadBalancerBackendAddressPools = &backendAddressPools if nicSpec.PublicIPName != "" { nicConfig.PublicIPAddress = &network.PublicIPAddress{ diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index b20c4f2ce6e..b1a48e055ab 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -42,10 +42,11 @@ func (s *Service) Reconcile(ctx context.Context) error { addressVersion = network.IPv6 } + // only set DNS properties if there is a DNS name specified var dnsSettings *network.PublicIPAddressDNSSettings if ip.DNSName != "" { dnsSettings = &network.PublicIPAddressDNSSettings{ - DomainNameLabel: to.StringPtr(strings.ToLower(ip.Name)), + DomainNameLabel: to.StringPtr(strings.Split(ip.DNSName, ".")[0]), Fqdn: to.StringPtr(ip.DNSName), } } @@ -82,6 +83,7 @@ func (s *Service) Delete(ctx context.Context) error { defer span.End() for _, ip := range s.Scope.PublicIPSpecs() { + // TODO: if supporting BYO IP make sure unmanaged IPs aren't deleted s.Scope.V(2).Info("deleting public IP", "public ip", ip.Name) err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), ip.Name) if err != nil && azure.ResourceNotFound(err) { diff --git a/cloud/services/routetables/mock_routetables/routetables_mock.go b/cloud/services/routetables/mock_routetables/routetables_mock.go index 74b80d039f6..cbaf2aa1156 100644 --- a/cloud/services/routetables/mock_routetables/routetables_mock.go +++ b/cloud/services/routetables/mock_routetables/routetables_mock.go @@ -398,6 +398,34 @@ func (mr *MockRouteTableScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockRouteTableScope)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockRouteTableScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockRouteTableScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockRouteTableScope)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockRouteTableScope) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockRouteTableScopeMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockRouteTableScope)(nil).NodeOutboundLBName)) +} + // RouteTableSpecs mocks base method. func (m *MockRouteTableScope) RouteTableSpecs() []azure.RouteTableSpec { m.ctrl.T.Helper() diff --git a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go index 868784faecb..8502d2dd6c9 100644 --- a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go +++ b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go @@ -398,6 +398,34 @@ func (mr *MockNSGScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockNSGScope)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockNSGScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockNSGScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockNSGScope)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockNSGScope) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockNSGScopeMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockNSGScope)(nil).NodeOutboundLBName)) +} + // NSGSpecs mocks base method. func (m *MockNSGScope) NSGSpecs() []azure.NSGSpec { m.ctrl.T.Helper() diff --git a/cloud/services/subnets/mock_subnets/subnets_mock.go b/cloud/services/subnets/mock_subnets/subnets_mock.go index e5f6a208978..ff222ac3b2d 100644 --- a/cloud/services/subnets/mock_subnets/subnets_mock.go +++ b/cloud/services/subnets/mock_subnets/subnets_mock.go @@ -398,6 +398,34 @@ func (mr *MockSubnetScopeMockRecorder) ControlPlaneRouteTable() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ControlPlaneRouteTable", reflect.TypeOf((*MockSubnetScope)(nil).ControlPlaneRouteTable)) } +// APIServerLBName mocks base method. +func (m *MockSubnetScope) APIServerLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBName indicates an expected call of APIServerLBName. +func (mr *MockSubnetScopeMockRecorder) APIServerLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockSubnetScope)(nil).APIServerLBName)) +} + +// NodeOutboundLBName mocks base method. +func (m *MockSubnetScope) NodeOutboundLBName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret0, _ := ret[0].(string) + return ret0 +} + +// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. +func (mr *MockSubnetScopeMockRecorder) NodeOutboundLBName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockSubnetScope)(nil).NodeOutboundLBName)) +} + // SubnetSpecs mocks base method. func (m *MockSubnetScope) SubnetSpecs() []azure.SubnetSpec { m.ctrl.T.Helper() diff --git a/cloud/services/subnets/subnets.go b/cloud/services/subnets/subnets.go index bd02e46adf4..efad4dce8a3 100644 --- a/cloud/services/subnets/subnets.go +++ b/cloud/services/subnets/subnets.go @@ -148,11 +148,10 @@ func (s *Service) getExisting(ctx context.Context, rgName string, spec azure.Sub } subnetSpec := &infrav1.SubnetSpec{ - Role: spec.Role, - InternalLBIPAddress: spec.InternalLBIPAddress, - Name: to.String(subnet.Name), - ID: to.String(subnet.ID), - CIDRBlocks: addresses, + Role: spec.Role, + Name: to.String(subnet.Name), + ID: to.String(subnet.ID), + CIDRBlocks: addresses, } return subnetSpec, nil diff --git a/cloud/types.go b/cloud/types.go index 98fa79efb42..983fadd8876 100644 --- a/cloud/types.go +++ b/cloud/types.go @@ -29,22 +29,20 @@ type PublicIPSpec struct { // NICSpec defines the specification for a Network Interface. type NICSpec struct { - Name string - MachineName string - SubnetName string - VNetName string - VNetResourceGroup string - StaticIPAddress string - PublicLBName string - PublicLBAddressPoolName string - PublicLBNATRuleName string - InternalLBName string - InternalLBAddressPoolName string - PublicIPName string - VMSize string - AcceleratedNetworking *bool - IPv6Enabled bool - EnableIPForwarding bool + Name string + MachineName string + SubnetName string + VNetName string + VNetResourceGroup string + StaticIPAddress string + LBName string + LBBackendAddressPoolName string + LBNATRuleName string + PublicIPName string + VMSize string + AcceleratedNetworking *bool + IPv6Enabled bool + EnableIPForwarding bool } // DiskSpec defines the specification for a Disk. @@ -54,13 +52,14 @@ type DiskSpec struct { // LBSpec defines the specification for a Load Balancer. type LBSpec struct { - Name string - PublicIPName string - Role string - SubnetName string - SubnetCidrs []string - PrivateIPAddress string - APIServerPort int32 + Name string + Role string + Type infrav1.LBType + SubnetName string + SubnetCidrs []string + BackendPoolName string + FrontendIPConfigs []infrav1.FrontendIP + APIServerPort int32 } // RouteTableRole defines the unique role of a route table. @@ -80,13 +79,12 @@ type InboundNatSpec struct { // SubnetSpec defines the specification for a Subnet. type SubnetSpec struct { - Name string - CIDRs []string - VNetName string - RouteTableName string - SecurityGroupName string - Role infrav1.SubnetRole - InternalLBIPAddress string + Name string + CIDRs []string + VNetName string + RouteTableName string + SecurityGroupName string + Role infrav1.SubnetRole } // VNetSpec defines the specification for a Virtual Network. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml index f45336f66fc..12c33351d81 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml @@ -465,6 +465,46 @@ spec: description: NetworkSpec encapsulates all things related to Azure network. properties: + apiServerLB: + description: APIServerLB is the configuration for the control-plane + load balancer. + properties: + frontendIPs: + items: + description: FrontendIP defines a load balancer frontend + IP configuration. + properties: + name: + minLength: 1 + type: string + privateIP: + type: string + publicIP: + description: PublicIPSpec defines the inputs to create + an Azure public IP address. + properties: + dnsName: + type: string + name: + type: string + required: + - name + type: object + required: + - name + type: object + type: array + id: + type: string + name: + type: string + sku: + description: SKU defines an Azure load balancer SKU. + type: string + type: + description: LBType defines an Azure load balancer Type. + type: string + type: object subnets: description: Subnets is the configuration for the control-plane subnet and the node subnet. @@ -487,9 +527,10 @@ spec: this resource. type: string internalLBIPAddress: - description: InternalLBIPAddress is the IP address that + description: 'InternalLBIPAddress is the IP address that will be used as the internal LB private IP. For the control - plane subnet only. + plane subnet only. Deprecated: Use LoadBalancer private + IP instead' type: string name: description: Name defines a name for the subnet resource. @@ -695,52 +736,6 @@ spec: This list will be used by Cluster API to try and spread the machines across the failure domains.' type: object - network: - description: Network encapsulates the state of Azure networking resources. - properties: - apiServerIp: - description: APIServerIP is the Kubernetes API server public IP - address. - properties: - dnsName: - type: string - id: - type: string - ipAddress: - type: string - name: - type: string - type: object - apiServerLb: - description: APIServerLB is the Kubernetes API server load balancer. - properties: - backendPool: - description: BackendPool defines a load balancer backend pool. - properties: - id: - type: string - name: - type: string - type: object - frontendIpConfig: - description: FrontendIPConfig - DO NOT USE this empty struct - is here to preserve backwards compatibility and should be - removed in v1alpha4 - type: object - id: - type: string - name: - type: string - sku: - description: SKU defines an Azure load balancer SKU. - type: string - tags: - additionalProperties: - type: string - description: Tags defines a map of tags. - type: object - type: object - type: object ready: description: Ready is true when the provider resource is ready. type: boolean diff --git a/controllers/azurecluster_controller.go b/controllers/azurecluster_controller.go index 225a7fb6aec..d255ce1927a 100644 --- a/controllers/azurecluster_controller.go +++ b/controllers/azurecluster_controller.go @@ -209,15 +209,9 @@ func (r *AzureClusterReconciler) reconcileNormal(ctx context.Context, clusterSco return reconcile.Result{}, wrappedErr } - if azureCluster.Status.Network.APIServerIP.DNSName == "" { - clusterScope.Info("Waiting for Load Balancer to exist") - conditions.MarkFalse(azureCluster, infrav1.NetworkInfrastructureReadyCondition, infrav1.LoadBalancerProvisioningReason, clusterv1.ConditionSeverityWarning, err.Error()) - return reconcile.Result{RequeueAfter: 15 * time.Second}, nil - } - // Set APIEndpoints so the Cluster API Cluster Controller can pull them azureCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ - Host: azureCluster.Status.Network.APIServerIP.DNSName, + Host: clusterScope.APIServerHost(), Port: clusterScope.APIServerPort(), } diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index d4ba8e59e1f..d419a7274ab 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -18,8 +18,6 @@ package controllers import ( "context" - "fmt" - "hash/fnv" "github.com/pkg/errors" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" @@ -70,14 +68,11 @@ func (r *azureClusterReconciler) Reconcile(ctx context.Context) error { ctx, span := tele.Tracer().Start(ctx, "controllers.azureClusterReconciler.Reconcile") defer span.End() - if err := r.createOrUpdateNetworkAPIServerIP(); err != nil { - return errors.Wrapf(err, "failed to create or update network API server IP for cluster %s in location %s", r.scope.ClusterName(), r.scope.Location()) - } - if err := r.setFailureDomainsForLocation(ctx); err != nil { return errors.Wrapf(err, "failed to get availability zones") } + r.scope.SetDNSName() r.scope.SetControlPlaneIngressRules() if err := r.groupsSvc.Reconcile(ctx); err != nil { @@ -149,20 +144,6 @@ func (r *azureClusterReconciler) Delete(ctx context.Context) error { return nil } -// CreateOrUpdateNetworkAPIServerIP creates or updates public ip name and dns name -func (r *azureClusterReconciler) createOrUpdateNetworkAPIServerIP() error { - if r.scope.Network().APIServerIP.Name == "" { - h := fnv.New32a() - if _, err := h.Write([]byte(fmt.Sprintf("%s/%s/%s", r.scope.SubscriptionID(), r.scope.ResourceGroup(), r.scope.ClusterName()))); err != nil { - return errors.Wrapf(err, "failed to write hash sum for api server ip") - } - r.scope.Network().APIServerIP.Name = azure.GeneratePublicIPName(r.scope.ClusterName(), fmt.Sprintf("%x", h.Sum32())) - } - - r.scope.Network().APIServerIP.DNSName = r.scope.GenerateFQDN() - return nil -} - func (r *azureClusterReconciler) setFailureDomainsForLocation(ctx context.Context) error { zones, err := r.skuCache.GetZones(ctx, r.scope.Location()) if err != nil { diff --git a/hack/debugging/kubectl-capz-ssh b/hack/debugging/kubectl-capz-ssh index bffc3883a68..8ddfb6b8604 100755 --- a/hack/debugging/kubectl-capz-ssh +++ b/hack/debugging/kubectl-capz-ssh @@ -24,7 +24,7 @@ if [[ "$1" == '--' ]]; then shift; fi echo "finding address for $azurecluster" capimachine=$(kubectl get azuremachine $azuremachine -o json | jq -r '.metadata.ownerReferences | .[] | select(.kind == "Machine").name') azurecluster=$(kubectl get machine $capimachine -o json | jq -r '.spec.clusterName') -apiserver=$(kubectl get azurecluster $azurecluster -o json | jq -r '.status.network.apiServerIp.dnsName') +apiserver=$(kubectl get azurecluster $azurecluster -o json | jq -r '.spec.controlPlaneEndpoint.host') echo "found address $apiserver" echo "finding address for $azuremachine" diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml deleted file mode 100644 index ab2bf7c478e..00000000000 --- a/templates/cluster-template.yaml +++ /dev/null @@ -1,199 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: Cluster -metadata: - labels: - cni: calico - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - location: ${AZURE_LOCATION} - networkSpec: - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - infrastructureTemplate: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - useExperimentalRetryJoin: true - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1alpha3 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - location: ${AZURE_LOCATION} - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - useExperimentalRetryJoin: true diff --git a/templates/test/prow-private/patches/custom-vnet.yaml b/templates/test/prow-private/patches/custom-vnet.yaml new file mode 100644 index 00000000000..e1cdfd52b00 --- /dev/null +++ b/templates/test/prow-private/patches/custom-vnet.yaml @@ -0,0 +1,23 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} +spec: + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + subnets: + - name: private-cp-subnet + role: control-plane + cidrBlocks: + - ${AZURE_CP_SUBNET_CIDR} + - name: private-node-subnet + role: node + cidrBlocks: + - ${AZURE_NODE_SUBNET_CIDR} + apiServerLB: + name: ${CLUSTER_NAME}-internal-lb + type: Internal + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-frontend + privateIP: ${AZURE_INTERNAL_LB_IP} From 8482bc80fc342e59bb75fab87ea8a165f5d4cc9a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Fri, 9 Oct 2020 15:02:00 -0700 Subject: [PATCH 2/7] Do not delete public IP for BYO API Server IP scenario --- api/v1alpha2/azurecluster_conversion.go | 2 + api/v1alpha3/azurecluster_default_test.go | 18 +- api/v1alpha3/azurecluster_validation.go | 6 +- api/v1alpha3/azurecluster_validation_test.go | 22 +- api/v1alpha3/azurecluster_webhook_test.go | 18 - cloud/scope/cluster.go | 3 +- cloud/services/loadbalancers/loadbalancers.go | 7 +- .../loadbalancers/loadbalancers_test.go | 373 ++++++------------ .../networkinterfaces_test.go | 107 +++-- cloud/services/publicips/publicips.go | 32 +- cloud/services/publicips/publicips_test.go | 107 ++++- cloud/services/subnets/subnets_test.go | 208 +++++----- cloud/types.go | 1 - 13 files changed, 446 insertions(+), 458 deletions(-) diff --git a/api/v1alpha2/azurecluster_conversion.go b/api/v1alpha2/azurecluster_conversion.go index 63ffaaafb4d..55ad1ef0eb6 100644 --- a/api/v1alpha2/azurecluster_conversion.go +++ b/api/v1alpha2/azurecluster_conversion.go @@ -74,6 +74,8 @@ func (src *AzureCluster) ConvertTo(dstRaw conversion.Hub) error { // nolint } } + dst.Spec.NetworkSpec.APIServerLB = restored.Spec.NetworkSpec.APIServerLB + // Manually convert conditions dst.SetConditions(restored.GetConditions()) diff --git a/api/v1alpha3/azurecluster_default_test.go b/api/v1alpha3/azurecluster_default_test.go index b4089199eef..e1dbb8db9ac 100644 --- a/api/v1alpha3/azurecluster_default_test.go +++ b/api/v1alpha3/azurecluster_default_test.go @@ -114,6 +114,20 @@ func TestVnetDefaults(t *testing.T) { RouteTable: RouteTable{}, }, }, + APIServerLB: LoadBalancerSpec{ + Name: "my-lb", + SKU: SKUStandard, + FrontendIPs: []FrontendIP{ + { + Name: "ip-config", + PublicIP: &PublicIPSpec{ + Name: "public-ip", + DNSName: "myfqdn.azure.com", + }, + }, + }, + Type: Public, + }, }, }, }, @@ -548,12 +562,12 @@ func TestAPIServerLBDefaults(t *testing.T) { NetworkSpec: NetworkSpec{ APIServerLB: LoadBalancerSpec{ Name: "cluster-test-public-lb", - SKU: SKUStandard, + SKU: SKUStandard, FrontendIPs: []FrontendIP{ { Name: "cluster-test-public-lb-frontEnd", PublicIP: &PublicIPSpec{ - Name: "pip-apiserver-cluster-test", + Name: "pip-cluster-test-apiserver", DNSName: "", }, }, diff --git a/api/v1alpha3/azurecluster_validation.go b/api/v1alpha3/azurecluster_validation.go index 0a8cd8f9ee9..99071160652 100644 --- a/api/v1alpha3/azurecluster_validation.go +++ b/api/v1alpha3/azurecluster_validation.go @@ -97,7 +97,11 @@ func validateNetworkSpec(networkSpec NetworkSpec, old NetworkSpec, fldPath *fiel } allErrs = append(allErrs, validateSubnets(networkSpec.Subnets, fldPath.Child("subnets"))...) } - allErrs = append(allErrs, validateAPIServerLB(networkSpec.APIServerLB, old.APIServerLB, networkSpec.GetControlPlaneSubnet().CIDRBlocks, fldPath.Child("apiServerLB"))...) + var cidrBlocks []string + if subnet := networkSpec.GetControlPlaneSubnet(); subnet != nil { + cidrBlocks = subnet.CIDRBlocks + } + allErrs = append(allErrs, validateAPIServerLB(networkSpec.APIServerLB, old.APIServerLB, cidrBlocks, fldPath.Child("apiServerLB"))...) if len(allErrs) == 0 { return nil } diff --git a/api/v1alpha3/azurecluster_validation_test.go b/api/v1alpha3/azurecluster_validation_test.go index 9b0bb1ced0b..628b29396da 100644 --- a/api/v1alpha3/azurecluster_validation_test.go +++ b/api/v1alpha3/azurecluster_validation_test.go @@ -754,7 +754,7 @@ func TestValidateAPIServerLB(t *testing.T) { name: "internal LB with in range private IP", lb: LoadBalancerSpec{ Type: Internal, - SKU: SKUStandard, + SKU: SKUStandard, Name: "my-private-lb", FrontendIPs: []FrontendIP{ { @@ -806,7 +806,8 @@ func createValidNetworkSpec() NetworkSpec { ResourceGroup: "custom-vnet", Name: "my-vnet", }, - Subnets: createValidSubnets(), + Subnets: createValidSubnets(), + APIServerLB: createValidAPIServerLB(), } } @@ -822,3 +823,20 @@ func createValidSubnets() Subnets { }, } } + +func createValidAPIServerLB() LoadBalancerSpec { + return LoadBalancerSpec{ + Name: "my-lb", + SKU: SKUStandard, + FrontendIPs: []FrontendIP{ + { + Name: "ip-config", + PublicIP: &PublicIPSpec{ + Name: "public-ip", + DNSName: "myfqdn.azure.com", + }, + }, + }, + Type: Public, + } +} diff --git a/api/v1alpha3/azurecluster_webhook_test.go b/api/v1alpha3/azurecluster_webhook_test.go index df1d8be4d67..ee25a804521 100644 --- a/api/v1alpha3/azurecluster_webhook_test.go +++ b/api/v1alpha3/azurecluster_webhook_test.go @@ -83,15 +83,6 @@ func TestAzureCluster_ValidateCreate(t *testing.T) { }(), wantErr: true, }, - { - name: "azurecluster with pre-existing vnet - invalid internal load balancer ip address", - cluster: func() *AzureCluster { - cluster := createValidCluster() - cluster.Spec.NetworkSpec.Subnets[0].InternalLBIPAddress = "1000.1.1.1" - return cluster - }(), - wantErr: true, - }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -166,15 +157,6 @@ func TestAzureCluster_ValidateUpdate(t *testing.T) { }(), wantErr: true, }, - { - name: "azurecluster with pre-existing vnet - invalid internal load balancer ip address", - cluster: func() *AzureCluster { - cluster := createValidCluster() - cluster.Spec.NetworkSpec.Subnets[0].InternalLBIPAddress = "1000.1.1.1" - return cluster - }(), - wantErr: true, - }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 19137966622..3265de79ca9 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -126,7 +126,6 @@ func (s *ClusterScope) LBSpecs() []azure.LBSpec { // Control Plane LB Name: s.APIServerLB().Name, SubnetName: s.ControlPlaneSubnet().Name, - SubnetCidrs: s.ControlPlaneSubnet().CIDRBlocks, FrontendIPConfigs: s.APIServerLB().FrontendIPs, APIServerPort: s.APIServerPort(), Type: s.APIServerLB().Type, @@ -420,7 +419,7 @@ func (s *ClusterScope) SetDNSName() { // for back compat, set the old API Server defaults if no API Server Spec has been set by new webhooks. lb := s.APIServerLB() if lb == nil || lb.Name == "" { - lbName := fmt.Sprintf("%s-%s", s.ClusterName(), "public-lb") + lbName := fmt.Sprintf("%s-%s", s.ClusterName(), "public-lb") ip, dns := s.GenerateLegacyFQDN() lb = &infrav1.LoadBalancerSpec{ Name: lbName, diff --git a/cloud/services/loadbalancers/loadbalancers.go b/cloud/services/loadbalancers/loadbalancers.go index 5c8e505f1f8..322c1930e65 100644 --- a/cloud/services/loadbalancers/loadbalancers.go +++ b/cloud/services/loadbalancers/loadbalancers.go @@ -43,7 +43,6 @@ func (s *Service) Reconcile(ctx context.Context) error { } lb := network.LoadBalancer{ - Name: to.StringPtr(lbSpec.Name), Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ @@ -116,6 +115,7 @@ func (s *Service) Reconcile(ctx context.Context) error { }, }, } + // TODO: check if this is needed for outbound access if lbSpec.Type == infrav1.Internal { lb.LoadBalancerPropertiesFormat.OutboundRules = nil } @@ -154,10 +154,7 @@ func (s *Service) Delete(ctx context.Context) error { } func (s *Service) getFrontendIPConfigs(lbSpec azure.LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource, error) { - ctx, span := tele.Tracer().Start(ctx, "loadbalancers.Service.getFrontendIPConfigs") - defer span.End() - - var frontendIPConfigurations []network.FrontendIPConfiguration + frontendIPConfigurations := make([]network.FrontendIPConfiguration, 0) frontendIDs := make([]network.SubResource, 0) for _, ipConfig := range lbSpec.FrontendIPConfigs { var properties network.FrontendIPConfigurationPropertiesFormat diff --git a/cloud/services/loadbalancers/loadbalancers_test.go b/cloud/services/loadbalancers/loadbalancers_test.go index 3423f9cd875..3eca4c79cdc 100644 --- a/cloud/services/loadbalancers/loadbalancers_test.go +++ b/cloud/services/loadbalancers/loadbalancers_test.go @@ -43,14 +43,13 @@ func TestReconcileLoadBalancer(t *testing.T) { }{ { name: "fail to create a public LB", - expectedError: "failed to create load balancer my-publiclb: #: Internal Server Error: StatusCode=500", + expectedError: "failed to create load balancer \"my-publiclb\": #: Internal Server Error: StatusCode=500", expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.LBSpecs().Return([]azure.LBSpec{ { - Name: "my-publiclb", - PublicIPName: "my-publicip", - Role: infrav1.APIServerRole, + Name: "my-publiclb", + Role: infrav1.APIServerRole, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -62,15 +61,26 @@ func TestReconcileLoadBalancer(t *testing.T) { }, }, { - name: "create apiserver LB", + name: "create public apiserver LB", expectedError: "", expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.LBSpecs().Return([]azure.LBSpec{ { - Name: "my-publiclb", - PublicIPName: "my-publicip", - Role: infrav1.APIServerRole, + Name: "my-publiclb", + Role: infrav1.APIServerRole, + Type: infrav1.Public, + SubnetName: "my-cp-subnet", + BackendPoolName: "my-publiclb-backendPool", + FrontendIPConfigs: []infrav1.FrontendIP{ + { + Name: "my-publiclb-frontEnd", + PublicIP: &infrav1.PublicIPSpec{ + Name: "my-publicip", + DNSName: "my-cluster.12345.mydomain.com", + }, + }, + }, APIServerPort: 6443, }, }) @@ -92,8 +102,7 @@ func TestReconcileLoadBalancer(t *testing.T) { { Name: to.StringPtr("my-publiclb-frontEnd"), FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Dynamic, - PublicIPAddress: &network.PublicIPAddress{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/publicIPAddresses/my-publicip")}, + PublicIPAddress: &network.PublicIPAddress{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/publicIPAddresses/my-publicip")}, }, }, }, @@ -157,57 +166,93 @@ func TestReconcileLoadBalancer(t *testing.T) { }, }, { - name: "create node outbound LB", + name: "create internal apiserver LB", expectedError: "", expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.LBSpecs().Return([]azure.LBSpec{ { - Name: "cluster-name", - PublicIPName: "outbound-publicip", - Role: infrav1.NodeOutboundRole, + Name: "my-private-lb", + Role: infrav1.APIServerRole, + Type: infrav1.Internal, + SubnetName: "my-cp-subnet", + BackendPoolName: "my-private-lb-backendPool", + FrontendIPConfigs: []infrav1.FrontendIP{ + { + Name: "my-private-lb-frontEnd", + PrivateIPAddress: "10.0.0.10", + }, + }, + APIServerPort: 6443, }, }) s.SubscriptionID().AnyTimes().Return("123") s.ResourceGroup().AnyTimes().Return("my-rg") + s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ + ResourceGroup: "my-rg", + Name: "my-vnet", + }) s.Location().AnyTimes().Return("testlocation") - s.ClusterName().AnyTimes().Return("cluster-name") + s.ClusterName().AnyTimes().Return("my-cluster") s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) gomock.InOrder( - m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "cluster-name", gomockinternal.DiffEq(network.LoadBalancer{ + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-private-lb", gomockinternal.DiffEq(network.LoadBalancer{ Tags: map[string]*string{ - "sigs.k8s.io_cluster-api-provider-azure_cluster_cluster-name": to.StringPtr("owned"), - "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.NodeOutboundRole), + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.APIServerRole), }, Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, Location: to.StringPtr("testlocation"), LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ { - Name: to.StringPtr("cluster-name-frontEnd"), + Name: to.StringPtr("my-private-lb-frontEnd"), FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Dynamic, - PublicIPAddress: &network.PublicIPAddress{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/publicIPAddresses/outbound-publicip")}, + PrivateIPAllocationMethod: network.Static, + Subnet: &network.Subnet{ + ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-cp-subnet"), + }, + PrivateIPAddress: to.StringPtr("10.0.0.10"), }, }, }, BackendAddressPools: &[]network.BackendAddressPool{ { - Name: to.StringPtr("cluster-name-outboundBackendPool"), + Name: to.StringPtr("my-private-lb-backendPool"), }, }, - OutboundRules: &[]network.OutboundRule{ + LoadBalancingRules: &[]network.LoadBalancingRule{ { - Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ - FrontendIPConfigurations: &[]network.SubResource{ - {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/frontendIPConfigurations/cluster-name-frontEnd")}, + Name: to.StringPtr("LBRuleHTTPS"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + DisableOutboundSnat: to.BoolPtr(true), + Protocol: network.TransportProtocolTCP, + FrontendPort: to.Int32Ptr(6443), + BackendPort: to.Int32Ptr(6443), + IdleTimeoutInMinutes: to.Int32Ptr(4), + EnableFloatingIP: to.BoolPtr(false), + LoadDistribution: network.LoadDistributionDefault, + FrontendIPConfiguration: &network.SubResource{ + ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-private-lb/frontendIPConfigurations/my-private-lb-frontEnd"), }, BackendAddressPool: &network.SubResource{ - ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/backendAddressPools/cluster-name-outboundBackendPool"), + ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-private-lb/backendAddressPools/my-private-lb-backendPool"), }, - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), + Probe: &network.SubResource{ + ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-private-lb/probes/HTTPSProbe"), + }, + }, + }, + }, + Probes: &[]network.Probe{ + { + Name: to.StringPtr("HTTPSProbe"), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: network.ProbeProtocolHTTPS, + Port: to.Int32Ptr(6443), + RequestPath: to.StringPtr("/healthz"), + IntervalInSeconds: to.Int32Ptr(15), + NumberOfProbes: to.Int32Ptr(4), }, }, }, @@ -216,182 +261,70 @@ func TestReconcileLoadBalancer(t *testing.T) { }, }, { - name: "internal load balancer does not exist", + name: "create node outbound LB", expectedError: "", expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.LBSpecs().Return([]azure.LBSpec{ { - Name: "my-lb", - SubnetCidrs: []string{"10.0.0.0/16"}, - SubnetName: "my-subnet", - PrivateIPAddress: "10.0.0.10", - Role: infrav1.InternalRole, - }, - }) - s.SubscriptionID().AnyTimes().Return("123") - s.ResourceGroup().AnyTimes().Return("my-rg") - s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ - ResourceGroup: "my-rg", - Name: "my-vnet", - }) - s.Location().AnyTimes().Return("testlocation") - s.ClusterName().AnyTimes().Return("cluster-name") - s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) - s.IsIPv6Enabled().AnyTimes().Return(false) - m.Get(gomockinternal.AContext(), "my-rg", "my-lb").Return(network.LoadBalancer{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) - mVnet.CheckIPAddressAvailability(gomockinternal.AContext(), "my-rg", "my-vnet", "10.0.0.10").Return(network.IPAddressAvailabilityResult{Available: to.BoolPtr(true)}, nil) - m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-lb", gomock.AssignableToTypeOf(network.LoadBalancer{})) - }, - }, - { - name: "internal load balancer retrieval fails", - expectedError: "failed to look for existing internal LB: #: Internal Server Error: StatusCode=500", - expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { - s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) - s.LBSpecs().Return([]azure.LBSpec{ - { - Name: "my-lb", - SubnetCidrs: []string{"10.0.0.0/16"}, - SubnetName: "my-subnet", - PrivateIPAddress: "10.0.0.10", - Role: infrav1.InternalRole, + Name: "cluster-name", + Role: infrav1.NodeOutboundRole, + Type: infrav1.Public, + BackendPoolName: "cluster-name-outboundBackendPool", + FrontendIPConfigs: []infrav1.FrontendIP{ + { + Name: "cluster-name-frontEnd", + PublicIP: &infrav1.PublicIPSpec{ + Name: "outbound-publicip", + }, + }, + }, }, }) s.SubscriptionID().AnyTimes().Return("123") s.ResourceGroup().AnyTimes().Return("my-rg") - s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ - ResourceGroup: "my-rg", - Name: "my-vnet", - }) s.Location().AnyTimes().Return("testlocation") s.ClusterName().AnyTimes().Return("cluster-name") s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) - m.Get(gomockinternal.AContext(), "my-rg", "my-lb").Return(network.LoadBalancer{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) - }, - }, - { - name: "internal load balancer exists", - expectedError: "", - expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { - s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) - s.LBSpecs().Return([]azure.LBSpec{ - { - Name: "my-lb", - SubnetCidrs: []string{"10.0.0.0/16"}, - SubnetName: "my-subnet", - PrivateIPAddress: "10.0.0.10", - Role: infrav1.InternalRole, - APIServerPort: 100, - }, - }) - s.SubscriptionID().AnyTimes().Return("123") - s.ResourceGroup().AnyTimes().Return("my-rg") - s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ - ResourceGroup: "my-rg", - Name: "my-vnet", - }) - s.Location().AnyTimes().Return("testlocation") - s.ClusterName().AnyTimes().Return("my-cluster") - s.IsIPv6Enabled().AnyTimes().Return(false) - s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) - m.Get(gomockinternal.AContext(), "my-rg", "my-lb").Return(network.LoadBalancer{ - LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ - { - FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAddress: to.StringPtr("10.0.0.10"), - PrivateIPAllocationMethod: network.Static, - }, - }, - }}}, nil) - m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-lb", gomockinternal.DiffEq(network.LoadBalancer{ - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, - Location: to.StringPtr("testlocation"), - Tags: map[string]*string{ - "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), - "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.InternalRole), - }, - LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ - { - Name: to.StringPtr("my-lb-frontEnd"), - FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Static, - PrivateIPAddress: to.StringPtr("10.0.0.10"), - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - }, - }, + gomock.InOrder( + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "cluster-name", gomockinternal.DiffEq(network.LoadBalancer{ + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_cluster-name": to.StringPtr("owned"), + "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.NodeOutboundRole), }, - Probes: &[]network.Probe{ - { - Name: to.StringPtr("HTTPSProbe"), - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolHTTPS, - RequestPath: to.StringPtr("/healthz"), - Port: to.Int32Ptr(100), - IntervalInSeconds: to.Int32Ptr(15), - NumberOfProbes: to.Int32Ptr(4), + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Location: to.StringPtr("testlocation"), + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: to.StringPtr("cluster-name-frontEnd"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PublicIPAddress: &network.PublicIPAddress{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/publicIPAddresses/outbound-publicip")}, + }, }, }, - }, - BackendAddressPools: &[]network.BackendAddressPool{ - { - Name: to.StringPtr("my-lb-backendPool"), + BackendAddressPools: &[]network.BackendAddressPool{ + { + Name: to.StringPtr("cluster-name-outboundBackendPool"), + }, }, - }, - LoadBalancingRules: &[]network.LoadBalancingRule{ - { - Name: to.StringPtr("LBRuleHTTPS"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - Protocol: network.TransportProtocolTCP, - FrontendPort: to.Int32Ptr(100), - BackendPort: to.Int32Ptr(100), - IdleTimeoutInMinutes: to.Int32Ptr(4), - EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, - FrontendIPConfiguration: &network.SubResource{ - ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-lb/frontendIPConfigurations/my-lb-frontEnd"), - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-lb/backendAddressPools/my-lb-backendPool"), - }, - Probe: &network.SubResource{ - ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-lb/probes/HTTPSProbe"), + OutboundRules: &[]network.OutboundRule{ + { + Name: to.StringPtr("OutboundNATAllProtocols"), + OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]network.SubResource{ + {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/frontendIPConfigurations/cluster-name-frontEnd")}, + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/backendAddressPools/cluster-name-outboundBackendPool"), + }, + Protocol: network.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(4), }, }, }, }, - }, - })) - }, - }, - { - name: "internal load balancer does not exist and IP is not available", - expectedError: "IP 10.0.0.10 is not available in VNet my-vnet and there were no other available IPs found", - expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, m *mock_loadbalancers.MockClientMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { - s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) - s.LBSpecs().Return([]azure.LBSpec{ - { - Name: "my-lb", - SubnetCidrs: []string{"10.0.0.0/16"}, - SubnetName: "my-subnet", - PrivateIPAddress: "10.0.0.10", - Role: infrav1.InternalRole, - }, - }) - s.SubscriptionID().AnyTimes().Return("123") - s.ResourceGroup().AnyTimes().Return("my-rg") - s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ - ResourceGroup: "my-rg", - Name: "my-vnet", - }) - s.Location().AnyTimes().Return("testlocation") - s.ClusterName().AnyTimes().Return("cluster-name") - s.IsIPv6Enabled().AnyTimes().Return(false) - s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) - m.Get(gomockinternal.AContext(), "my-rg", "my-lb").Return(network.LoadBalancer{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) - mVnet.CheckIPAddressAvailability(gomockinternal.AContext(), "my-rg", "my-vnet", "10.0.0.10").Return(network.IPAddressAvailabilityResult{Available: to.BoolPtr(false)}, nil) + })).Return(nil)) }, }, { @@ -401,23 +334,22 @@ func TestReconcileLoadBalancer(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.LBSpecs().Return([]azure.LBSpec{ { - Name: "my-lb", - SubnetCidrs: []string{"10.0.0.0/16"}, - SubnetName: "my-subnet", - PrivateIPAddress: "10.0.0.10", - APIServerPort: 6443, - Role: infrav1.InternalRole, + Name: "my-lb", + SubnetName: "my-subnet", + APIServerPort: 6443, + Role: infrav1.APIServerRole, + Type: infrav1.Internal, }, { Name: "my-lb-2", APIServerPort: 6443, - PublicIPName: "my-apiserver-ip", Role: infrav1.APIServerRole, + Type: infrav1.Public, }, { - Name: "my-lb-3", - PublicIPName: "my-node-ip", - Role: infrav1.NodeOutboundRole, + Name: "my-lb-3", + Role: infrav1.NodeOutboundRole, + Type: infrav1.Public, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -430,8 +362,6 @@ func TestReconcileLoadBalancer(t *testing.T) { s.ClusterName().AnyTimes().Return("cluster-name") s.IsIPv6Enabled().AnyTimes().Return(false) s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) - m.Get(gomockinternal.AContext(), "my-rg", "my-lb").Return(network.LoadBalancer{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) - mVnet.CheckIPAddressAvailability(gomockinternal.AContext(), "my-rg", "my-vnet", "10.0.0.10").Return(network.IPAddressAvailabilityResult{Available: to.BoolPtr(true)}, nil) m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-lb", gomock.AssignableToTypeOf(network.LoadBalancer{})) m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-lb-2", gomock.AssignableToTypeOf(network.LoadBalancer{})) m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-lb-3", gomock.AssignableToTypeOf(network.LoadBalancer{})) @@ -459,7 +389,6 @@ func TestReconcileLoadBalancer(t *testing.T) { Client: clientMock, VirtualNetworksClient: vnetMock, } - err := s.Reconcile(context.TODO()) if tc.expectedError != "" { g.Expect(err).To(HaveOccurred()) @@ -556,49 +485,3 @@ func TestDeleteLoadBalancer(t *testing.T) { }) } } - -func TestGetAvailablePrivateIP(t *testing.T) { - g := NewWithT(t) - - testcases := []struct { - name string - subnetCidrs []string - expectedIP string - expect func(s *mock_loadbalancers.MockLBScopeMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) - }{ - { - name: "internal load balancer with a valid subnet cidr", - subnetCidrs: []string{"10.0.8.0/16"}, - expectedIP: "10.0.8.0", - expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { - mVnet.CheckIPAddressAvailability(gomockinternal.AContext(), "my-rg", "my-vnet", "10.0.8.0").Return(network.IPAddressAvailabilityResult{Available: to.BoolPtr(true)}, nil) - }, - }, - { - name: "internal load balancer subnet cidr not 8 characters in length", - subnetCidrs: []string{"10.64.8.0"}, - expectedIP: "10.64.8.0", - expect: func(s *mock_loadbalancers.MockLBScopeMockRecorder, mVnet *mock_virtualnetworks.MockClientMockRecorder) { - mVnet.CheckIPAddressAvailability(gomockinternal.AContext(), "my-rg", "my-vnet", "10.64.8.0").Return(network.IPAddressAvailabilityResult{Available: to.BoolPtr(true)}, nil) - }, - }, - } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - mockCtrl := gomock.NewController(t) - scopeMock := mock_loadbalancers.NewMockLBScope(mockCtrl) - vnetMock := mock_virtualnetworks.NewMockClient(mockCtrl) - - tc.expect(scopeMock.EXPECT(), vnetMock.EXPECT()) - - s := &Service{ - Scope: scopeMock, - VirtualNetworksClient: vnetMock, - } - - resultIP, err := s.getAvailablePrivateIP(context.TODO(), "my-rg", "my-vnet", "", tc.subnetCidrs) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(resultIP).To(Equal(tc.expectedIP)) - }) - } -} diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index 0ec6717c4eb..733037daf05 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -19,7 +19,9 @@ package networkinterfaces import ( "context" "fmt" + "github.com/google/go-cmp/cmp" "net/http" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" "testing" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" @@ -30,11 +32,9 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" - "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" "k8s.io/klog/klogr" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" ) func TestReconcileNetworkInterface(t *testing.T) { @@ -84,7 +84,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", + LBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, }, @@ -105,16 +105,16 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", - PublicLBAddressPoolName: "cluster-name-outboundBackendPool", - StaticIPAddress: "fake.static.ip", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + LBName: "my-public-lb", + LBBackendAddressPoolName: "cluster-name-outboundBackendPool", + StaticIPAddress: "fake.static.ip", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -150,15 +150,15 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", - PublicLBAddressPoolName: "cluster-name-outboundBackendPool", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + LBName: "my-public-lb", + LBBackendAddressPoolName: "cluster-name-outboundBackendPool", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -193,18 +193,16 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", - PublicLBAddressPoolName: "my-public-lb-backendPool", - PublicLBNATRuleName: "azure-test1", - InternalLBName: "my-internal-lb", - InternalLBAddressPoolName: "my-internal-lb-backendPool", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + LBName: "my-public-lb", + LBBackendAddressPoolName: "my-public-lb-backendPool", + LBNATRuleName: "azure-test1", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -227,7 +225,7 @@ func TestReconcileNetworkInterface(t *testing.T) { LoadBalancerInboundNatRules: &[]network.InboundNatRule{{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-public-lb/inboundNatRules/azure-test1")}}, LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{ {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-public-lb/backendAddressPools/my-public-lb-backendPool")}, - {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-internal-lb/backendAddressPools/my-internal-lb-backendPool")}}, + }, }, }, }, @@ -271,7 +269,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", + LBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, }, @@ -291,9 +289,8 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, - LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, }, }, }, @@ -313,7 +310,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", + LBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: to.BoolPtr(false), }, @@ -334,9 +331,8 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, - LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, }, }, }, @@ -356,7 +352,7 @@ func TestReconcileNetworkInterface(t *testing.T) { VNetName: "my-vnet", IPv6Enabled: true, VNetResourceGroup: "my-rg", - PublicLBName: "my-public-lb", + LBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, EnableIPForwarding: true, @@ -378,9 +374,8 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, - LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, }, }, { @@ -462,9 +457,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - PublicLBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + LBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -478,9 +473,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - PublicLBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + LBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -495,9 +490,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - PublicLBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + LBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.ResourceGroup().AnyTimes().Return("my-rg") diff --git a/cloud/services/publicips/publicips.go b/cloud/services/publicips/publicips.go index b1a48e055ab..2c3449ace39 100644 --- a/cloud/services/publicips/publicips.go +++ b/cloud/services/publicips/publicips.go @@ -18,6 +18,8 @@ package publicips import ( "context" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + "sigs.k8s.io/cluster-api-provider-azure/cloud/converters" "strings" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" @@ -56,6 +58,12 @@ func (s *Service) Reconcile(ctx context.Context) error { s.Scope.ResourceGroup(), ip.Name, network.PublicIPAddress{ + Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ + ClusterName: s.Scope.ClusterName(), + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: to.StringPtr(ip.Name), + Additional: s.Scope.AdditionalTags(), + })), Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, Name: to.StringPtr(ip.Name), Location: to.StringPtr(s.Scope.Location()), @@ -83,9 +91,18 @@ func (s *Service) Delete(ctx context.Context) error { defer span.End() for _, ip := range s.Scope.PublicIPSpecs() { - // TODO: if supporting BYO IP make sure unmanaged IPs aren't deleted + managed, err := s.isIPManaged(ctx, ip.Name) + if err != nil && !azure.ResourceNotFound(err) { + return errors.Wrap(err, "could not get public IP management state") + } + + if !managed { + s.Scope.V(2).Info("Skipping IP deletion for unmanaged public IP", "public ip", ip.Name) + continue + } + s.Scope.V(2).Info("deleting public IP", "public ip", ip.Name) - err := s.Client.Delete(ctx, s.Scope.ResourceGroup(), ip.Name) + err = s.Client.Delete(ctx, s.Scope.ResourceGroup(), ip.Name) if err != nil && azure.ResourceNotFound(err) { // already deleted continue @@ -98,3 +115,14 @@ func (s *Service) Delete(ctx context.Context) error { } return nil } + +// isIPManaged returns true if the IP has an owned tag with the cluster name as value, +// meaning that the IP's lifecycle is managed. +func (s *Service) isIPManaged(ctx context.Context, ipName string) (bool, error) { + ip, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), ipName) + if err != nil { + return false, err + } + tags := converters.MapToTags(ip.Tags) + return tags.HasOwned(s.Scope.ClusterName()), nil +} diff --git a/cloud/services/publicips/publicips_test.go b/cloud/services/publicips/publicips_test.go index 2362e754900..487093859ed 100644 --- a/cloud/services/publicips/publicips_test.go +++ b/cloud/services/publicips/publicips_test.go @@ -21,6 +21,8 @@ import ( "net/http" "testing" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" @@ -56,11 +58,11 @@ func TestReconcilePublicIP(t *testing.T) { s.PublicIPSpecs().Return([]azure.PublicIPSpec{ { Name: "my-publicip", - DNSName: "fakedns", + DNSName: "fakedns.mydomain.io", }, { Name: "my-publicip-2", - DNSName: "fakedns2", + DNSName: "fakedns2-52959.uksouth.cloudapp.azure.com", }, { Name: "my-publicip-3", @@ -68,22 +70,28 @@ func TestReconcilePublicIP(t *testing.T) { { Name: "my-publicip-ipv6", IsIPv6: true, - DNSName: "fakename", + DNSName: "fakename.mydomain.io", }, }) s.ResourceGroup().AnyTimes().Return("my-rg") + s.ClusterName().AnyTimes().Return("my-cluster") + s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) s.Location().AnyTimes().Return("testlocation") gomock.InOrder( m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-publicip", gomockinternal.DiffEq(network.PublicIPAddress{ Name: to.StringPtr("my-publicip"), Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, Location: to.StringPtr("testlocation"), + Tags: map[string]*string{ + "Name": to.StringPtr("my-publicip"), + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + }, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAddressVersion: network.IPv4, PublicIPAllocationMethod: network.Static, DNSSettings: &network.PublicIPAddressDNSSettings{ - DomainNameLabel: to.StringPtr("my-publicip"), - Fqdn: to.StringPtr("fakedns"), + DomainNameLabel: to.StringPtr("fakedns"), + Fqdn: to.StringPtr("fakedns.mydomain.io"), }, }, })).Times(1), @@ -91,12 +99,16 @@ func TestReconcilePublicIP(t *testing.T) { Name: to.StringPtr("my-publicip-2"), Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, Location: to.StringPtr("testlocation"), + Tags: map[string]*string{ + "Name": to.StringPtr("my-publicip-2"), + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + }, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAddressVersion: network.IPv4, PublicIPAllocationMethod: network.Static, DNSSettings: &network.PublicIPAddressDNSSettings{ - DomainNameLabel: to.StringPtr("my-publicip-2"), - Fqdn: to.StringPtr("fakedns2"), + DomainNameLabel: to.StringPtr("fakedns2-52959"), + Fqdn: to.StringPtr("fakedns2-52959.uksouth.cloudapp.azure.com"), }, }, })).Times(1), @@ -104,6 +116,10 @@ func TestReconcilePublicIP(t *testing.T) { Name: to.StringPtr("my-publicip-3"), Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, Location: to.StringPtr("testlocation"), + Tags: map[string]*string{ + "Name": to.StringPtr("my-publicip-3"), + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + }, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAddressVersion: network.IPv4, PublicIPAllocationMethod: network.Static, @@ -113,12 +129,16 @@ func TestReconcilePublicIP(t *testing.T) { Name: to.StringPtr("my-publicip-ipv6"), Sku: &network.PublicIPAddressSku{Name: network.PublicIPAddressSkuNameStandard}, Location: to.StringPtr("testlocation"), + Tags: map[string]*string{ + "Name": to.StringPtr("my-publicip-ipv6"), + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + }, PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ PublicIPAddressVersion: network.IPv6, PublicIPAllocationMethod: network.Static, DNSSettings: &network.PublicIPAddressDNSSettings{ - DomainNameLabel: to.StringPtr("my-publicip-ipv6"), - Fqdn: to.StringPtr("fakename"), + DomainNameLabel: to.StringPtr("fakename"), + Fqdn: to.StringPtr("fakename.mydomain.io"), }, }, })).Times(1), @@ -133,10 +153,12 @@ func TestReconcilePublicIP(t *testing.T) { s.PublicIPSpecs().Return([]azure.PublicIPSpec{ { Name: "my-publicip", - DNSName: "fakedns", + DNSName: "fakedns.mydomain.io", }, }) s.ResourceGroup().AnyTimes().Return("my-rg") + s.ClusterName().AnyTimes().Return("my-cluster") + s.AdditionalTags().AnyTimes().Return(infrav1.Tags{}) s.Location().AnyTimes().Return("testlocation") m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-publicip", gomock.AssignableToTypeOf(network.PublicIPAddress{})).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) }, @@ -192,7 +214,22 @@ func TestDeletePublicIP(t *testing.T) { }, }) s.ResourceGroup().AnyTimes().Return("my-rg") + s.ClusterName().AnyTimes().Return("my-cluster") + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-publicip"), + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "foo": to.StringPtr("bar"), + }, + }, nil) m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip") + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip-2").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-publicip-2"), + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "foo": to.StringPtr("buzz"), + }, + }, nil) m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip-2") }, }, @@ -210,8 +247,15 @@ func TestDeletePublicIP(t *testing.T) { }, }) s.ResourceGroup().AnyTimes().Return("my-rg") - m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip"). - Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) + s.ClusterName().AnyTimes().Return("my-cluster") + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip").Return(network.PublicIPAddress{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 404}, "Not found")) + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip-2").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-public-ip-2"), + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "foo": to.StringPtr("buzz"), + }, + }, nil) m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip-2") }, }, @@ -226,10 +270,49 @@ func TestDeletePublicIP(t *testing.T) { }, }) s.ResourceGroup().AnyTimes().Return("my-rg") + s.ClusterName().AnyTimes().Return("my-cluster") + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-publicip"), + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "foo": to.StringPtr("bar"), + }, + }, nil) m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip"). Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) }, }, + { + name: "skip unmanaged public ip deletion", + expectedError: "", + expect: func(s *mock_publicips.MockPublicIPScopeMockRecorder, m *mock_publicips.MockClientMockRecorder) { + s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) + s.PublicIPSpecs().Return([]azure.PublicIPSpec{ + { + Name: "my-publicip", + }, + { + Name: "my-publicip-2", + }, + }) + s.ResourceGroup().AnyTimes().Return("my-rg") + s.ClusterName().AnyTimes().Return("my-cluster") + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-public-ip"), + Tags: map[string]*string{ + "foo": to.StringPtr("bar"), + }, + }, nil) + m.Get(gomockinternal.AContext(), "my-rg", "my-publicip-2").Return(network.PublicIPAddress{ + Name: to.StringPtr("my-publicip-2"), + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_my-cluster": to.StringPtr("owned"), + "foo": to.StringPtr("buzz"), + }, + }, nil) + m.Delete(gomockinternal.AContext(), "my-rg", "my-publicip-2") + }, + }, } for _, tc := range testcases { diff --git a/cloud/services/subnets/subnets_test.go b/cloud/services/subnets/subnets_test.go index c837c95ecef..b928a38b759 100644 --- a/cloud/services/subnets/subnets_test.go +++ b/cloud/services/subnets/subnets_test.go @@ -49,13 +49,12 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -82,13 +81,12 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-ipv6-subnet", - CIDRs: []string{"10.0.0.0/16", "2001:1234:5678:9abd::/64"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-ipv6-subnet", + CIDRs: []string{"10.0.0.0/16", "2001:1234:5678:9abd::/64"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet", ResourceGroup: "my-rg"}) @@ -117,13 +115,12 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -144,13 +141,12 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -168,13 +164,12 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "custom-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "custom-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", Name: "custom-vnet", ID: "id1", Tags: infrav1.Tags{ @@ -197,22 +192,20 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().AnyTimes().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, { - Name: "my-subnet-1", - CIDRs: []string{"10.2.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg-1", - Role: infrav1.SubnetControlPlane, - InternalLBIPAddress: "10.2.0.20", + Name: "my-subnet-1", + CIDRs: []string{"10.2.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg-1", + Role: infrav1.SubnetControlPlane, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -268,22 +261,20 @@ func TestReconcileSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().AnyTimes().Return([]azure.SubnetSpec{ { - Name: "my-ipv6-subnet", - CIDRs: []string{"10.0.0.0/16", "2001:1234:5678:9abd::/64"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-ipv6-subnet", + CIDRs: []string{"10.0.0.0/16", "2001:1234:5678:9abd::/64"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, { - Name: "my-ipv6-subnet-cp", - CIDRs: []string{"10.2.0.0/16", "2001:1234:5678:9abc::/64"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg-1", - Role: infrav1.SubnetControlPlane, - InternalLBIPAddress: "10.2.0.20", + Name: "my-ipv6-subnet-cp", + CIDRs: []string{"10.2.0.0/16", "2001:1234:5678:9abc::/64"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg-1", + Role: infrav1.SubnetControlPlane, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -383,22 +374,20 @@ func TestDeleteSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, { - Name: "my-subnet-1", - CIDRs: []string{"10.1.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetControlPlane, - InternalLBIPAddress: "10.1.0.20", + Name: "my-subnet-1", + CIDRs: []string{"10.1.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetControlPlane, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -415,13 +404,12 @@ func TestDeleteSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -438,22 +426,20 @@ func TestDeleteSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, { - Name: "my-subnet-1", - CIDRs: []string{"10.1.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetControlPlane, - InternalLBIPAddress: "10.1.0.20", + Name: "my-subnet-1", + CIDRs: []string{"10.1.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetControlPlane, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet"}) @@ -471,13 +457,12 @@ func TestDeleteSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "custom-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "custom-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{ResourceGroup: "custom-vnet-rg", Name: "custom-vnet", ID: "id1"}) @@ -492,13 +477,12 @@ func TestDeleteSubnets(t *testing.T) { s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) s.SubnetSpecs().Return([]azure.SubnetSpec{ { - Name: "my-subnet", - CIDRs: []string{"10.0.0.0/16"}, - VNetName: "my-vnet", - RouteTableName: "my-subnet_route_table", - SecurityGroupName: "my-sg", - Role: infrav1.SubnetNode, - InternalLBIPAddress: "10.0.0.10", + Name: "my-subnet", + CIDRs: []string{"10.0.0.0/16"}, + VNetName: "my-vnet", + RouteTableName: "my-subnet_route_table", + SecurityGroupName: "my-sg", + Role: infrav1.SubnetNode, }, }) s.Vnet().AnyTimes().Return(&infrav1.VnetSpec{Name: "my-vnet", ResourceGroup: "my-rg"}) diff --git a/cloud/types.go b/cloud/types.go index 983fadd8876..df1c8cdc8ed 100644 --- a/cloud/types.go +++ b/cloud/types.go @@ -56,7 +56,6 @@ type LBSpec struct { Role string Type infrav1.LBType SubnetName string - SubnetCidrs []string BackendPoolName string FrontendIPConfigs []infrav1.FrontendIP APIServerPort int32 From aec76b10d5055790ba2387192ba61d561c372d0a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Fri, 9 Oct 2020 15:46:54 -0700 Subject: [PATCH 3/7] Add api server lb documentation --- docs/book/src/SUMMARY.md | 1 + docs/book/src/topics/api-server-endpoint.md | 105 ++++++++++++++++++++ docs/book/src/topics/custom-vnet.md | 31 +----- 3 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 docs/book/src/topics/api-server-endpoint.md diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index 9f240773be0..e286f0de5b0 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -3,6 +3,7 @@ [Introduction](./introduction.md) [Roadmap](./roadmap.md) - [Topics](./topics/topics.md) + - [API Server Endpoint](./topics/api-server-endpoint.md) - [Cloud Provider Config](./topics/cloud-provider-config.md) - [Custom Images](./topics/custom-images.md) - [Data Disks](./topics/data-disks.md) diff --git a/docs/book/src/topics/api-server-endpoint.md b/docs/book/src/topics/api-server-endpoint.md new file mode 100644 index 00000000000..c2b3870d998 --- /dev/null +++ b/docs/book/src/topics/api-server-endpoint.md @@ -0,0 +1,105 @@ +# API Server Endpoint + +This document describes how to configure your clusters' api server load balancer and IP. + +### Load Balancer Type + +CAPZ supports two load balancer types, `Public` and `Internal`. `Public`, which is also the default, means that your API Server Load Balancer will have a publicly accessible IP address. +`Internal`, also known as a "private cluster", means that the API Server endpoint will only be accessible from within the cluster's virtual network (or peered VNets). + +A `Public` cluster will have an Azure public load balancer load balancing internet traffic to the control plane nodes. + +A `Private` cluster will have an Azure internal load balancer load balancing traffic inside the VNet to the control plane nodes. + +For more information on Azure load balancing, see [Load Balancer documentation](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-overview). + + + +Here is an example of configuring the API Server LB type: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: my-private-cluster + namespace: default +spec: + location: eastus + networkSpec: + apiServerLB: + type: Internal +``` + +### Private IP + +When using an api server load balancer of type `Internal`, the default private IP address associated with that load balancer will be `10.0.0.100`. +If also specifying a [custom virtual network](./custom-vnet.md), make sure you provide a private IP address that is in the range of your control plane subnet and not in use. + +For example: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: my-private-cluster + namespace: default +spec: + location: eastus + networkSpec: + vnet: + name: my-vnet + cidrBlocks: + - 172.16.0.0/16 + subnets: + - name: my-subnet-cp + role: control-plane + cidrBlocks: + - 172.16.0.0/24 + - name: my-subnet-node + role: node + cidrBlocks: + - 172.16.2.0/24 + apiServerLB: + type: Internal + frontendIPs: + - name: lb-private-ip-frontend + privateIP: 172.16.0.100 +``` + +### Public IP + +When using an api server load balancer of type `Public`, a dynamic public IP address will be created, along with a unique FQDN. + +You can also choose to provide your own public api server IP. To do so, specify the existing public IP as follows: + +````yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: my-cluster + namespace: default +spec: + location: eastus + networkSpec: + apiServerLB: + type: Public + frontendIPs: + - name: lb-public-ip-frontend + publicIP: + name: my-public-ip + dns: my-cluster-986b4408.eastus.cloudapp.azure.com +```` + +Note that `dns` is the FQDN associated to your public IP address (look for "DNS name" in the Azure Portal). + +When you BYO api server IP, CAPZ does not manage its lifecycle, ie. the IP will not get deleted as part of cluster deletion. + +### Load Balancer SKU + +At this time, CAPZ only supports Azure Standard Load Balancers. See [SKU comparison](https://docs.microsoft.com/en-us/azure/load-balancer/skus#skus) for more information on Azure Load Balancers SKUs. diff --git a/docs/book/src/topics/custom-vnet.md b/docs/book/src/topics/custom-vnet.md index 674acd6ecfc..3004611a19c 100644 --- a/docs/book/src/topics/custom-vnet.md +++ b/docs/book/src/topics/custom-vnet.md @@ -5,7 +5,7 @@ To deploy a cluster using a pre-existing vnet, modify the `AzureCluster` spec to include the name and resource group of the existing vnet as follows, as well as the control plane and node subnets as follows: ```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureCluster metadata: name: cluster-byo-vnet @@ -24,30 +24,7 @@ spec: resourceGroup: cluster-byo-vnet ``` -When providing a vnet, it is required to also provide the two subnets that should be used for control planes and nodes. The internal load balancer private IP can be optionally provided in the control plane subnet spec as follows: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 -kind: AzureCluster -metadata: - name: cluster-byo-vnet - namespace: default -spec: - location: southcentralus - networkSpec: - vnet: - resourceGroup: custom-vnet - name: my-vnet - subnets: - - name: control-plane-subnet - role: control-plane - internalLBIPAddress: "10.0.1.6" - - name: node-subnet - role: node - resourceGroup: cluster-byo-vnet -``` - -If provided, the private IP should be a valid IP within the control plane subnet address space. If no IP is provided, the internal load balancer reconciler will select a free IP within the subnet range at creation. +When providing a vnet, it is required to also provide the two subnets that should be used for control planes and nodes. If providing an existing vnet and subnets with existing network security groups, make sure that the control plane security group allows inbound to port 6443, as port 6443 is used by kubeadm to bootstrap the control planes. Alternatively, you can [provide a custom control plane endpoint](https://github.com/kubernetes-sigs/cluster-api-bootstrap-provider-kubeadm#kubeadmconfig-objects) in the `KubeadmConfig` spec. @@ -58,7 +35,7 @@ The pre-existing vnet can be in the same resource group or a different resource It is also possible to customize the vnet to be created without providing an already existing vnet. To do so, simply modify the `AzureCluster` `NetworkSpec` as desired. Here is an illustrative example of a cluster with a customized vnet address space (CIDR) and customized subnets: ```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureCluster metadata: name: cluster-example @@ -95,7 +72,7 @@ It is the responsibility of the user to supply those rules themselves if using c Here is an illustrative example of customizing ingresses that builds on the one above by adding an ingress rule to the control plane nodes: ```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 kind: AzureCluster metadata: name: cluster-example From bdb8fc3d912b51a03553b038d18911808f15e40a Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 15 Oct 2020 14:55:27 -0700 Subject: [PATCH 4/7] Add private cluster E2E test Update azure_test.go --- Makefile | 4 +- cloud/defaults.go | 4 +- templates/cluster-template-private.yaml | 214 ++++++++++++++++ templates/flavors/private/kustomization.yaml | 6 + .../flavors/private/patches/private-lb.yaml | 23 ++ .../test/cluster-template-prow-private.yaml | 238 ++++++++++++++++++ .../test/prow-private/cni-resource-set.yaml | 21 ++ .../test/prow-private/kustomization.yaml | 12 + .../prow-private/patches/cluster-cni.yaml | 8 + test/e2e/azure_privatecluster.go | 227 +++++++++++++++++ test/e2e/azure_test.go | 212 +++++++++------- test/e2e/common.go | 6 + test/e2e/config/azure-dev.yaml | 2 + 13 files changed, 880 insertions(+), 97 deletions(-) create mode 100644 templates/cluster-template-private.yaml create mode 100644 templates/flavors/private/kustomization.yaml create mode 100644 templates/flavors/private/patches/private-lb.yaml create mode 100644 templates/test/cluster-template-prow-private.yaml create mode 100644 templates/test/prow-private/cni-resource-set.yaml create mode 100644 templates/test/prow-private/kustomization.yaml create mode 100644 templates/test/prow-private/patches/cluster-cni.yaml create mode 100644 test/e2e/azure_privatecluster.go diff --git a/Makefile b/Makefile index 52105bfa3a4..0455e625403 100644 --- a/Makefile +++ b/Makefile @@ -519,9 +519,9 @@ delete-cluster: delete-workload-cluster ## Deletes the example kind cluster "ca kind delete cluster --name=capz .PHONY: kind-reset -kind-reset: ## Destroys the "capz" kind cluster. +kind-reset: ## Destroys the "capz" and "capz-e2e" kind clusters. kind delete cluster --name=capz || true - + kind delete cluster --name=capz-e2e || true ## -------------------------------------- ## Cleanup / Verification diff --git a/cloud/defaults.go b/cloud/defaults.go index b02caf47dcc..262bce2a818 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -105,8 +105,8 @@ func RouteTableID(subscriptionID, resourceGroup, routeTableName string) string { } // SecurityGroupID returns the azure resource ID for a given security group. -func SecurityGroupID(subscriptionID, resourceGroup, routeTableName string) string { - return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s", subscriptionID, resourceGroup, routeTableName) +func SecurityGroupID(subscriptionID, resourceGroup, nsgName string) string { + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s", subscriptionID, resourceGroup, nsgName) } // NetworkInterfaceID returns the azure resource ID for a given network interface. diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml new file mode 100644 index 00000000000..8f994ee39c1 --- /dev/null +++ b/templates/cluster-template-private.yaml @@ -0,0 +1,214 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-frontend + privateIP: 10.128.0.100 + name: ${CLUSTER_NAME}-internal-lb + type: Internal + subnets: + - cidrBlocks: + - ${AZURE_CP_SUBNET_CIDR} + name: private-cp-subnet + role: control-plane + - cidrBlocks: + - ${AZURE_NODE_SUBNET_CIDR} + name: private-node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml new file mode 100644 index 00000000000..f154854590a --- /dev/null +++ b/templates/flavors/private/kustomization.yaml @@ -0,0 +1,6 @@ +namespace: default +resources: + - ../default +patchesStrategicMerge: + - patches/private-lb.yaml + diff --git a/templates/flavors/private/patches/private-lb.yaml b/templates/flavors/private/patches/private-lb.yaml new file mode 100644 index 00000000000..ad558c8d8da --- /dev/null +++ b/templates/flavors/private/patches/private-lb.yaml @@ -0,0 +1,23 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} +spec: + networkSpec: + vnet: + name: ${AZURE_VNET_NAME} + subnets: + - name: private-cp-subnet + role: control-plane + cidrBlocks: + - ${AZURE_CP_SUBNET_CIDR} + - name: private-node-subnet + role: node + cidrBlocks: + - ${AZURE_NODE_SUBNET_CIDR} + apiServerLB: + name: ${CLUSTER_NAME}-internal-lb + type: Internal + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-frontend + privateIP: 10.128.0.100 \ No newline at end of file diff --git a/templates/test/cluster-template-prow-private.yaml b/templates/test/cluster-template-prow-private.yaml new file mode 100644 index 00000000000..2dc26d00665 --- /dev/null +++ b/templates/test/cluster-template-prow-private.yaml @@ -0,0 +1,238 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: ${CLUSTER_NAME}-crs-0 + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + additionalTags: + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + location: ${AZURE_LOCATION} + networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-frontend + privateIP: 10.128.0.100 + name: ${CLUSTER_NAME}-internal-lb + type: Internal + subnets: + - cidrBlocks: + - ${AZURE_CP_SUBNET_CIDR} + name: private-cp-subnet + role: control-plane + - cidrBlocks: + - ${AZURE_NODE_SUBNET_CIDR} + name: private-node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-private/cni-resource-set.yaml b/templates/test/prow-private/cni-resource-set.yaml new file mode 100644 index 00000000000..3bd1c9ac49c --- /dev/null +++ b/templates/test/prow-private/cni-resource-set.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +data: ${CNI_RESOURCES} +kind: ConfigMap +metadata: + name: cni-${CLUSTER_NAME}-crs-0 + namespace: default +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha3 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cni: ${CLUSTER_NAME}-crs-0 + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-crs-0 + strategy: ApplyOnce diff --git a/templates/test/prow-private/kustomization.yaml b/templates/test/prow-private/kustomization.yaml new file mode 100644 index 00000000000..582a3761411 --- /dev/null +++ b/templates/test/prow-private/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../flavors/private + - cni-resource-set.yaml +patchesStrategicMerge: + - ../patches/tags.yaml + - patches/cluster-cni.yaml + - patches/custom-vnet.yaml + + diff --git a/templates/test/prow-private/patches/cluster-cni.yaml b/templates/test/prow-private/patches/cluster-cni.yaml new file mode 100644 index 00000000000..751f46e6154 --- /dev/null +++ b/templates/test/prow-private/patches/cluster-cni.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default + labels: + cni: "${CLUSTER_NAME}-crs-0" diff --git a/test/e2e/azure_privatecluster.go b/test/e2e/azure_privatecluster.go new file mode 100644 index 00000000000..715363024c1 --- /dev/null +++ b/test/e2e/azure_privatecluster.go @@ -0,0 +1,227 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network" + "github.com/Azure/azure-sdk-for-go/profiles/latest/resources/mgmt/resources" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + "os" + "path/filepath" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AzurePrivateClusterSpecInput is the input for AzurePrivateClusterSpec. +type AzurePrivateClusterSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string + ClusterctlConfigPath string + E2EConfig *clusterctl.E2EConfig + ArtifactFolder string +} + +// AzurePrivateClusterSpec implements a test that creates a workload cluster with a private API endpoint. +func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivateClusterSpecInput) { + var ( + specName = "azure-private-cluster" + input AzurePrivateClusterSpecInput + publicClusterProxy framework.ClusterProxy + publicNamespace *corev1.Namespace + publicCancelWatches context.CancelFunc + cluster *clusterv1.Cluster + clusterName string + ) + + input = inputGetter() + Expect(input).ToNot(BeNil()) + Expect(input.BootstrapClusterProxy).NotTo(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).NotTo(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + By("creating a Kubernetes client to the workload cluster") + publicClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(context.TODO(), input.Namespace.Name, input.ClusterName) + + Byf("Creating a namespace for hosting the %s test spec", specName) + publicNamespace, publicCancelWatches = framework.CreateNamespaceAndWatchEvents(context.TODO(), framework.CreateNamespaceAndWatchEventsInput{ + Creator: publicClusterProxy.GetClient(), + ClientSet: publicClusterProxy.GetClientSet(), + Name: input.Namespace.Name, + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.ClusterName), + }) + + Expect(publicNamespace).NotTo(BeNil()) + Expect(publicCancelWatches).NotTo(BeNil()) + + By("Initializing the workload cluster") + clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterProxy: publicClusterProxy, + ClusterctlConfigPath: input.ClusterctlConfigPath, + InfrastructureProviders: input.E2EConfig.InfrastructureProviders(), + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.ClusterName), + }, input.E2EConfig.GetIntervals(specName, "wait-controllers")...) + + By("Ensure public API server is stable before creating private cluster") + Consistently(func() error { + kubeSystem := &corev1.Namespace{} + return publicClusterProxy.GetClient().Get(ctx, client.ObjectKey{Name: "kube-system"}, kubeSystem) + }, "5s", "100ms").Should(BeNil(), "Failed to assert public API server stability") + + By("Creating a private workload cluster") + clusterName = fmt.Sprintf("capz-e2e-%s", util.RandomString(6)) + Expect(os.Setenv(AzureResourceGroup, input.ClusterName)).NotTo(HaveOccurred()) + Expect(os.Setenv(AzureVNetName, fmt.Sprintf("%s-vnet", input.ClusterName))).NotTo(HaveOccurred()) + result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: publicClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", publicClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: publicClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "private", + Namespace: input.Namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + cluster = result.Cluster + + Expect(cluster).ToNot(BeNil()) +} + +// SetupExistingVNet creates a resource group and a VNet to be used by a workload cluster. +func SetupExistingVNet(ctx context.Context, vnetCidr string, cpSubnetCidrs, nodeSubnetCidrs map[string]string) { + By("creating Azure clients with the workload cluster's subscription") + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + authorizer, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + groupClient := resources.NewGroupsClient(subscriptionID) + groupClient.Authorizer = authorizer + vnetClient := network.NewVirtualNetworksClient(subscriptionID) + vnetClient.Authorizer = authorizer + nsgClient := network.NewSecurityGroupsClient(subscriptionID) + nsgClient.Authorizer = authorizer + + By("creating a resource group") + groupName := os.Getenv(AzureResourceGroup) + _, err = groupClient.CreateOrUpdate(ctx, groupName, resources.Group{ + Location: pointer.StringPtr(os.Getenv(AzureLocation)), + Tags: map[string]*string{ + "jobName": pointer.StringPtr(os.Getenv(JobName)), + "creationTimestamp": pointer.StringPtr(os.Getenv(Timestamp)), + }, + }) + Expect(err).To(BeNil()) + + By("creating a network security group") + nsgName := "control-plane-nsg" + securityRules := []network.SecurityRule{ + { + Name: pointer.StringPtr("allow_ssh"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Description: pointer.StringPtr("Allow SSH"), + Priority: pointer.Int32Ptr(2200), + Protocol: network.SecurityRuleProtocolTCP, + Access: network.SecurityRuleAccessAllow, + Direction: network.SecurityRuleDirectionInbound, + SourceAddressPrefix: pointer.StringPtr("*"), + SourcePortRange: pointer.StringPtr("*"), + DestinationAddressPrefix: pointer.StringPtr("*"), + DestinationPortRange: pointer.StringPtr("22"), + }, + }, + { + Name: pointer.StringPtr("allow_apiserver"), + SecurityRulePropertiesFormat: &network.SecurityRulePropertiesFormat{ + Description: pointer.StringPtr("Allow API Server"), + SourcePortRange: pointer.StringPtr("*"), + DestinationPortRange: pointer.StringPtr("6443"), + SourceAddressPrefix: pointer.StringPtr("*"), + DestinationAddressPrefix: pointer.StringPtr("*"), + Protocol: network.SecurityRuleProtocolTCP, + Access: network.SecurityRuleAccessAllow, + Direction: network.SecurityRuleDirectionInbound, + Priority: pointer.Int32Ptr(2201), + }, + }, + } + nsgFuture, err := nsgClient.CreateOrUpdate(ctx, groupName, nsgName, network.SecurityGroup{ + Location: pointer.StringPtr(os.Getenv(AzureLocation)), + SecurityGroupPropertiesFormat: &network.SecurityGroupPropertiesFormat{ + SecurityRules: &securityRules, + }, + }) + Expect(err).To(BeNil()) + err = nsgFuture.WaitForCompletionRef(ctx, nsgClient.Client) + Expect(err).To(BeNil()) + + By("creating a virtual network") + var subnets []network.Subnet + for name, cidr := range cpSubnetCidrs { + subnets = append(subnets, network.Subnet{ + SubnetPropertiesFormat: &network.SubnetPropertiesFormat{ + AddressPrefix: pointer.StringPtr(cidr), + NetworkSecurityGroup: &network.SecurityGroup{ + ID: pointer.StringPtr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/networkSecurityGroups/%s", subscriptionID, groupName, nsgName)), + }, + }, + Name: pointer.StringPtr(name), + }) + } + for name, cidr := range nodeSubnetCidrs { + subnets = append(subnets, network.Subnet{ + SubnetPropertiesFormat: &network.SubnetPropertiesFormat{ + AddressPrefix: pointer.StringPtr(cidr), + }, + Name: pointer.StringPtr(name), + }) + } + + vnetFuture, err := vnetClient.CreateOrUpdate(ctx, groupName, os.Getenv(AzureVNetName), network.VirtualNetwork{ + Location: pointer.StringPtr(os.Getenv(AzureLocation)), + VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{ + AddressSpace: &network.AddressSpace{ + AddressPrefixes: &[]string{vnetCidr}, + }, + Subnets: &subnets, + }, + }) + if err != nil { + fmt.Print(err.Error()) + } + Expect(err).To(BeNil()) + err = vnetFuture.WaitForCompletionRef(ctx, vnetClient.Client) + Expect(err).To(BeNil()) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 21e56f7cc7c..9218fc83b85 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -75,111 +75,137 @@ var _ = Describe("Workload cluster creation", func() { logCheckpoint(specTimes) }) - Context("Creating a single control-plane cluster", func() { - It("With 1 worker node", func() { - result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: bootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - ClusterctlConfigPath: clusterctlConfigPath, - KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: clusterctl.DefaultFlavor, - Namespace: namespace.Name, - ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(1), - WorkerMachineCount: pointer.Int64Ptr(1), - }, - WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), - }) - cluster = result.Cluster - - Context("Validating time synchronization", func() { - AzureTimeSyncSpec(ctx, func() AzureTimeSyncSpecInput { - return AzureTimeSyncSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } + if os.Getenv("LOCAL_ONLY") != "true" { + Context("Creating a private cluster", func() { + It("Creates a public management cluster in the same vnet", func() { + Context("Creating a custom virtual network", func() { + Expect(os.Setenv(AzureVNetName, "custom-vnet")).NotTo(HaveOccurred()) + cpCIDR := "10.128.0.0/16" + Expect(os.Setenv(AzureCPSubnetCidr, cpCIDR)).NotTo(HaveOccurred()) + nodeCIDR := "10.129.0.0/16" + Expect(os.Setenv(AzureNodeSubnetCidr, nodeCIDR)).NotTo(HaveOccurred()) + SetupExistingVNet(ctx, + "10.0.0.0/8", + map[string]string{fmt.Sprintf("%s-controlplane-subnet", clusterName): "10.0.0.0/16", "private-cp-subnet": cpCIDR}, + map[string]string{fmt.Sprintf("%s-node-subnet", clusterName): "10.1.0.0/16", "private-node-subnet": nodeCIDR}) }) - }) - }) - }) - Context("Creating a highly available control-plane cluster", func() { - It("With 3 control-plane nodes and 2 worker nodes", func() { - result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ - ClusterProxy: bootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - ClusterctlConfigPath: clusterctlConfigPath, - KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: clusterctl.DefaultFlavor, - Namespace: namespace.Name, - ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), - WorkerMachineCount: pointer.Int64Ptr(2), - }, - WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), - }) - cluster = result.Cluster + result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + cluster = result.Cluster + + Context("Validating time synchronization", func() { + AzureTimeSyncSpec(ctx, func() AzureTimeSyncSpecInput { + return AzureTimeSyncSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) - Context("Validating time synchronization", func() { - AzureTimeSyncSpec(ctx, func() AzureTimeSyncSpecInput { - return AzureTimeSyncSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } + Context("Creating a private cluster from the management cluster", func() { + AzurePrivateClusterSpec(ctx, func() AzurePrivateClusterSpecInput { + return AzurePrivateClusterSpecInput{ + Namespace: namespace, + ClusterName: clusterName, + ClusterctlConfigPath: clusterctlConfigPath, + E2EConfig: e2eConfig, + ArtifactFolder: artifactFolder, + } + }) }) }) + }) + } else { + Skip("test requires pushing container images to external repository") + } + + It("With 3 control-plane nodes and 2 worker nodes", func() { + result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(2), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + cluster = result.Cluster + + Context("Validating time synchronization", func() { + AzureTimeSyncSpec(ctx, func() AzureTimeSyncSpecInput { + return AzureTimeSyncSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) - Context("Validating failure domains", func() { - AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { - return AzureFailureDomainsSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Cluster: result.Cluster, - Namespace: namespace, - ClusterName: clusterName, - } - }) + Context("Validating failure domains", func() { + AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { + return AzureFailureDomainsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Cluster: result.Cluster, + Namespace: namespace, + ClusterName: clusterName, + } }) + }) - Context("Creating an accessible load balancer", func() { - AzureLBSpec(ctx, func() AzureLBSpecInput { - return AzureLBSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - } - }) + Context("Creating an accessible load balancer", func() { + AzureLBSpec(ctx, func() AzureLBSpecInput { + return AzureLBSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } }) + }) - Context("Validating network policies", func() { - AzureNetPolSpec(ctx, func() AzureNetPolSpecInput { - return AzureNetPolSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - } - }) + Context("Validating network policies", func() { + AzureNetPolSpec(ctx, func() AzureNetPolSpecInput { + return AzureNetPolSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } }) + }) - Context("Validating accelerated networking", func() { - AzureAcceleratedNetworkingSpec(ctx, func() AzureAcceleratedNetworkingSpecInput { - return AzureAcceleratedNetworkingSpecInput{ - ClusterName: clusterName, - } - }) + Context("Validating accelerated networking", func() { + AzureAcceleratedNetworkingSpec(ctx, func() AzureAcceleratedNetworkingSpecInput { + return AzureAcceleratedNetworkingSpecInput{ + ClusterName: clusterName, + } }) }) }) diff --git a/test/e2e/common.go b/test/e2e/common.go index bab5db306ce..f361fa09d7b 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -41,11 +41,17 @@ import ( // Test suite constants for e2e config variables const ( RedactLogScriptPath = "REDACT_LOG_SCRIPT" + AzureLocation = "AZURE_LOCATION" AzureResourceGroup = "AZURE_RESOURCE_GROUP" AzureVNetName = "AZURE_VNET_NAME" + AzureInternalLBIP = "AZURE_INTERNAL_LB_IP" + AzureCPSubnetCidr = "AZURE_CP_SUBNET_CIDR" + AzureNodeSubnetCidr = "AZURE_NODE_SUBNET_CIDR" CNIPathIPv6 = "CNI_IPV6" CNIResourcesIPv6 = "CNI_RESOURCES_IPV6" VMSSHPort = "VM_SSH_PORT" + JobName = "JOB_NAME" + Timestamp = "TIMESTAMP" ) func Byf(format string, a ...interface{}) { diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index ed699f29fb1..a1c6559609e 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -56,6 +56,8 @@ providers: targetName: "cluster-template-machine-pool.yaml" - sourcePath: "${PWD}/templates/test/cluster-template-prow-nvidia-gpu.yaml" targetName: "cluster-template-nvidia-gpu.yaml" + - sourcePath: "${PWD}/templates/test/cluster-template-prow-private.yaml" + targetName: "cluster-template-private.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.19.3}" From 8f7acd9e51635bd8200dc93bcd1f0e524966abf7 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Mon, 19 Oct 2020 17:31:51 -0700 Subject: [PATCH 5/7] Add control plane outbound lb Update cluster.go attach VMs to outbound LB fix PublicLBAddressPoolName address comments Update networkinterfaces_test.go address comments cleanup OutboundPoolName/APIServerLBName fix tests --- api/v1alpha3/azurecluster_default.go | 18 +-- api/v1alpha3/azurecluster_default_test.go | 39 ++++++- api/v1alpha3/tags.go | 3 + api/v1alpha3/types.go | 2 +- cloud/converters/loadbalancers.go | 31 +++++ cloud/defaults.go | 10 ++ cloud/interfaces.go | 5 +- cloud/mocks/service_mock.go | 108 ++++++++++++++++-- cloud/scope/cluster.go | 91 ++++++++++++--- cloud/scope/machine.go | 33 +++--- cloud/scope/machinepool.go | 4 +- cloud/scope/managedcontrolplane.go | 27 +++++ .../mocks_bastionhosts/bastionhosts_mock.go | 54 ++++++++- cloud/services/loadbalancers/loadbalancers.go | 3 +- .../loadbalancers/loadbalancers_test.go | 3 + .../mock_loadbalancers/loadbalancers_mock.go | 54 ++++++++- .../networkinterfaces/networkinterfaces.go | 25 ++-- .../networkinterfaces_test.go | 107 ++++++++--------- .../mock_routetables/routetables_mock.go | 54 ++++++++- .../securitygroups_mock.go | 54 ++++++++- .../subnets/mock_subnets/subnets_mock.go | 54 ++++++++- cloud/types.go | 31 ++--- 22 files changed, 636 insertions(+), 174 deletions(-) create mode 100644 cloud/converters/loadbalancers.go diff --git a/api/v1alpha3/azurecluster_default.go b/api/v1alpha3/azurecluster_default.go index cfaed82388f..c6a7ce4bce0 100644 --- a/api/v1alpha3/azurecluster_default.go +++ b/api/v1alpha3/azurecluster_default.go @@ -31,17 +31,6 @@ const ( DefaultInternalLBIPAddress = "10.0.0.100" ) -const ( - // DefaultVnetIPv6CIDR is the ipv6 Vnet CIDR - DefaultVnetIPv6CIDR = "2001:1234:5678:9a00::/56" - // DefaultControlPlaneSubnetIPv6CIDR is the default Control Plane Subnet CIDR - DefaultControlPlaneSubnetIPv6CIDR = "2001:1234:5678:9abc::/64" - // DefaultNodeSubnetIPv6CIDR is the default Node Subnet CIDR - DefaultNodeSubnetIPv6CIDR = "2001:1234:5678:9abd::/64" - // DefaultInternalLBIPv6Address is the default internal load balancer ip address - DefaultInternalLBIPv6Address = "2001:1234:5678:9abc::100" -) - func (c *AzureCluster) setDefaults() { c.setResourceGroupDefault() c.setNetworkSpecDefaults() @@ -137,15 +126,10 @@ func (c *AzureCluster) setAPIServerLBDefaults() { lb.Name = generateInternalLBName(c.ObjectMeta.Name) } if len(lb.FrontendIPs) == 0 { - // for back compat, set the private IP to the subnet InternalLBIPAddress value. - privateIP := c.Spec.NetworkSpec.GetControlPlaneSubnet().InternalLBIPAddress - if privateIP == "" { - privateIP = DefaultInternalLBIPAddress - } lb.FrontendIPs = []FrontendIP{ { Name: generateFrontendIPConfigName(lb.Name), - PrivateIPAddress: privateIP, + PrivateIPAddress: DefaultInternalLBIPAddress, }, } } diff --git a/api/v1alpha3/azurecluster_default_test.go b/api/v1alpha3/azurecluster_default_test.go index e1dbb8db9ac..023c8d98c9f 100644 --- a/api/v1alpha3/azurecluster_default_test.go +++ b/api/v1alpha3/azurecluster_default_test.go @@ -199,7 +199,7 @@ func TestVnetDefaults(t *testing.T) { ResourceGroup: "cluster-test", NetworkSpec: NetworkSpec{ Vnet: VnetSpec{ - CIDRBlocks: []string{DefaultVnetCIDR, DefaultVnetIPv6CIDR}, + CIDRBlocks: []string{DefaultVnetCIDR, "2001:1234:5678:9a00::/56"}, }, }, }, @@ -214,7 +214,7 @@ func TestVnetDefaults(t *testing.T) { Vnet: VnetSpec{ ResourceGroup: "cluster-test", Name: "cluster-test-vnet", - CIDRBlocks: []string{DefaultVnetCIDR, DefaultVnetIPv6CIDR}, + CIDRBlocks: []string{DefaultVnetCIDR, "2001:1234:5678:9a00::/56"}, }, }, }, @@ -578,6 +578,41 @@ func TestAPIServerLBDefaults(t *testing.T) { }, }, }, + { + name: "internal lb", + cluster: &AzureCluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "cluster-test", + }, + Spec: AzureClusterSpec{ + NetworkSpec: NetworkSpec{ + APIServerLB: LoadBalancerSpec{ + Type: Internal, + }, + }, + }, + }, + output: &AzureCluster{ + ObjectMeta: v1.ObjectMeta{ + Name: "cluster-test", + }, + Spec: AzureClusterSpec{ + NetworkSpec: NetworkSpec{ + APIServerLB: LoadBalancerSpec{ + Name: "cluster-test-internal-lb", + SKU: SKUStandard, + FrontendIPs: []FrontendIP{ + { + Name: "cluster-test-internal-lb-frontEnd", + PrivateIPAddress: DefaultInternalLBIPAddress, + }, + }, + Type: Internal, + }, + }, + }, + }, + }, } for _, c := range cases { diff --git a/api/v1alpha3/tags.go b/api/v1alpha3/tags.go index 0c8587b567d..fd8f64910c1 100644 --- a/api/v1alpha3/tags.go +++ b/api/v1alpha3/tags.go @@ -110,6 +110,9 @@ const ( // NodeOutboundRole describes the value for the node outbound LB role NodeOutboundRole = "nodeOutbound" + // ControlPlaneOutboundRole describes the value for the control plane outbound LB role + ControlPlaneOutboundRole = "controlPlaneOutbound" + // BastionRole describes the value for the bastion role BastionRole = "bastion" diff --git a/api/v1alpha3/types.go b/api/v1alpha3/types.go index 2c59f0c23b9..f6f64e02371 100644 --- a/api/v1alpha3/types.go +++ b/api/v1alpha3/types.go @@ -161,7 +161,7 @@ const ( type LBType string const ( - // Internal is the value for the Azure load b alancer internal type. + // Internal is the value for the Azure load balancer internal type. Internal = LBType("Internal") // Public is the value for the Azure load balancer public type. Public = LBType("Public") diff --git a/cloud/converters/loadbalancers.go b/cloud/converters/loadbalancers.go new file mode 100644 index 00000000000..eaeb62c779b --- /dev/null +++ b/cloud/converters/loadbalancers.go @@ -0,0 +1,31 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package converters + +import ( + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" +) + +// SKUtoSDK converts infrav1.SKU into a network.LoadBalancerSkuName. +func SKUtoSDK(src infrav1.SKU) network.LoadBalancerSkuName { + switch src { + case infrav1.SKUStandard: + return network.LoadBalancerSkuNameStandard + } + return "" +} diff --git a/cloud/defaults.go b/cloud/defaults.go index 262bce2a818..bf388aab4e7 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -64,6 +64,16 @@ func GenerateNodePublicIPName(machineName string) string { return fmt.Sprintf("pip-%s", machineName) } +// GenerateControlPlaneOutboundLBName generates the name of the control plane outbound LB. +func GenerateControlPlaneOutboundLBName(clusterName string) string { + return fmt.Sprintf("%s-outbound-lb", clusterName) +} + +// GenerateControlPlaneOutboundIPName generates a public IP name, based on the cluster name. +func GenerateControlPlaneOutboundIPName(clusterName string) string { + return fmt.Sprintf("pip-%s-controlplane-outbound", clusterName) +} + // GenerateNICName generates the name of a network interface based on the name of a VM. func GenerateNICName(machineName string) string { return fmt.Sprintf("%s-nic", machineName) diff --git a/cloud/interfaces.go b/cloud/interfaces.go index f925ca0dd13..4b473728fbb 100644 --- a/cloud/interfaces.go +++ b/cloud/interfaces.go @@ -66,7 +66,10 @@ type NetworkDescriber interface { NodeRouteTable() *infrav1.RouteTable ControlPlaneRouteTable() *infrav1.RouteTable APIServerLBName() string - NodeOutboundLBName() string + APIServerLBPoolName(string) string + IsAPIServerPrivate() bool + OutboundLBName(string) string + OutboundPoolName(string) string } // ClusterDescriber is an interface which can get common Azure Cluster information. diff --git a/cloud/mocks/service_mock.go b/cloud/mocks/service_mock.go index 1154997cca5..e099792cab4 100644 --- a/cloud/mocks/service_mock.go +++ b/cloud/mocks/service_mock.go @@ -452,18 +452,60 @@ func (mr *MockNetworkDescriberMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockNetworkDescriber)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockNetworkDescriber) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockNetworkDescriber) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockNetworkDescriberMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockNetworkDescriberMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockNetworkDescriber)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockNetworkDescriber)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockNetworkDescriber) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockNetworkDescriberMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockNetworkDescriber)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockNetworkDescriber) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockNetworkDescriberMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockNetworkDescriber)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockNetworkDescriber) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockNetworkDescriberMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockNetworkDescriber)(nil).OutboundPoolName), arg0) } // MockClusterDescriber is a mock of ClusterDescriber interface. @@ -932,16 +974,58 @@ func (mr *MockClusterScoperMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockClusterScoper)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockClusterScoper) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockClusterScoper) APIServerLBPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockClusterScoperMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockClusterScoper)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockClusterScoper) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockClusterScoperMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockClusterScoper)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockClusterScoper) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockClusterScoperMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockClusterScoper)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockClusterScoper) OutboundPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockClusterScoperMockRecorder) NodeOutboundLBName() *gomock.Call { +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockClusterScoperMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockClusterScoper)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockClusterScoper)(nil).OutboundPoolName), arg0) } diff --git a/cloud/scope/cluster.go b/cloud/scope/cluster.go index 3265de79ca9..687ba280432 100644 --- a/cloud/scope/cluster.go +++ b/cloud/scope/cluster.go @@ -104,24 +104,30 @@ func (s *ClusterScope) Authorizer() autorest.Authorizer { // PublicIPSpecs returns the public IP specs. func (s *ClusterScope) PublicIPSpecs() []azure.PublicIPSpec { - specs := []azure.PublicIPSpec{ - { - Name: azure.GenerateNodeOutboundIPName(s.NodeOutboundLBName()), - }, - } - if s.APIServerLB().Type == infrav1.Public { - specs = append(specs, azure.PublicIPSpec{ + var controlPlaneOutboundIP azure.PublicIPSpec + if s.IsAPIServerPrivate() { + controlPlaneOutboundIP = azure.PublicIPSpec{ + Name: azure.GenerateControlPlaneOutboundIPName(s.ClusterName()), + } + } else { + controlPlaneOutboundIP = azure.PublicIPSpec{ Name: s.APIServerPublicIP().Name, DNSName: s.APIServerPublicIP().DNSName, IsIPv6: false, // currently azure requires a ipv4 lb rule to enable ipv6 - }) + } + } + + return []azure.PublicIPSpec{ + controlPlaneOutboundIP, + { + Name: azure.GenerateNodeOutboundIPName(s.ClusterName()), + }, } - return specs } // LBSpecs returns the load balancer specs. func (s *ClusterScope) LBSpecs() []azure.LBSpec { - return []azure.LBSpec{ + specs := []azure.LBSpec{ { // Control Plane LB Name: s.APIServerLB().Name, @@ -129,8 +135,9 @@ func (s *ClusterScope) LBSpecs() []azure.LBSpec { FrontendIPConfigs: s.APIServerLB().FrontendIPs, APIServerPort: s.APIServerPort(), Type: s.APIServerLB().Type, + SKU: infrav1.SKUStandard, Role: infrav1.APIServerRole, - BackendPoolName: azure.GenerateBackendAddressPoolName(s.APIServerLB().Name), + BackendPoolName: s.APIServerLBPoolName(s.APIServerLB().Name), }, { // Public Node outbound LB @@ -139,15 +146,39 @@ func (s *ClusterScope) LBSpecs() []azure.LBSpec { { Name: azure.GenerateFrontendIPConfigName(s.NodeOutboundLBName()), PublicIP: &infrav1.PublicIPSpec{ - Name: azure.GenerateNodeOutboundIPName(s.NodeOutboundLBName()), + Name: azure.GenerateNodeOutboundIPName(s.ClusterName()), }, }, }, Type: infrav1.Public, - BackendPoolName: azure.GenerateOutboundBackendAddressPoolName(s.NodeOutboundLBName()), + SKU: infrav1.SKUStandard, + BackendPoolName: s.OutboundPoolName(s.NodeOutboundLBName()), Role: infrav1.NodeOutboundRole, }, } + + if !s.IsAPIServerPrivate() { + return specs + } + + specs = append(specs, azure.LBSpec{ + // Public Control Plane outbound LB + Name: azure.GenerateControlPlaneOutboundLBName(s.ClusterName()), + FrontendIPConfigs: []infrav1.FrontendIP{ + { + Name: azure.GenerateFrontendIPConfigName(azure.GenerateControlPlaneOutboundLBName(s.ClusterName())), + PublicIP: &infrav1.PublicIPSpec{ + Name: azure.GenerateControlPlaneOutboundIPName(s.ClusterName()), + }, + }, + }, + Type: infrav1.Public, + SKU: infrav1.SKUStandard, + BackendPoolName: s.OutboundPoolName(azure.GenerateControlPlaneOutboundLBName(s.ClusterName())), + Role: infrav1.ControlPlaneOutboundRole, + }) + + return specs } // RouteTableSpecs returns the node route table @@ -264,6 +295,11 @@ func (s *ClusterScope) APIServerLBName() string { return s.APIServerLB().Name } +// IsAPIServerPrivate returns true if the API Server LB is of type Internal. +func (s *ClusterScope) IsAPIServerPrivate() bool { + return s.APIServerLB().Type == infrav1.Internal +} + // APIServerPublicIP returns the API Server public IP. func (s *ClusterScope) APIServerPublicIP() *infrav1.PublicIPSpec { return s.APIServerLB().FrontendIPs[0].PublicIP @@ -274,11 +310,32 @@ func (s *ClusterScope) APIServerPrivateIP() string { return s.APIServerLB().FrontendIPs[0].PrivateIPAddress } +// APIServerLBPoolName returns the API Server LB backend pool name. +func (s *ClusterScope) APIServerLBPoolName(loadBalancerName string) string { + return azure.GenerateBackendAddressPoolName(loadBalancerName) +} + // NodeOutboundLBName returns the name of the node outbound LB. func (s *ClusterScope) NodeOutboundLBName() string { return s.ClusterName() } +// OutboundLBName returns the name of the outbound LB. +func (s *ClusterScope) OutboundLBName(role string) string { + if role == infrav1.Node { + return s.ClusterName() + } + if s.IsAPIServerPrivate() { + return azure.GenerateControlPlaneOutboundLBName(s.ClusterName()) + } + return s.APIServerLBName() +} + +// OutboundPoolName returns the outbound LB backend pool name. +func (s *ClusterScope) OutboundPoolName(loadBalancerName string) string { + return azure.GenerateOutboundBackendAddressPoolName(loadBalancerName) +} + // ResourceGroup returns the cluster resource group. func (s *ClusterScope) ResourceGroup() string { return s.AzureCluster.Spec.ResourceGroup @@ -372,7 +429,7 @@ func (s *ClusterScope) APIServerPort() int32 { // APIServerHost returns the APIServerHost used to reach the API server. func (s *ClusterScope) APIServerHost() string { - if s.APIServerLB().Type == infrav1.Internal { + if s.IsAPIServerPrivate() { return s.APIServerPrivateIP() } return s.APIServerPublicIP().DNSName @@ -438,9 +495,7 @@ func (s *ClusterScope) SetDNSName() { lb.DeepCopyInto(s.APIServerLB()) } // Generate valid FQDN if not set. - if s.APIServerLB().Type == infrav1.Public { - if s.APIServerPublicIP().DNSName == "" { - s.APIServerPublicIP().DNSName = s.GenerateFQDN(s.APIServerPublicIP().Name) - } + if !s.IsAPIServerPrivate() && s.APIServerPublicIP().DNSName == "" { + s.APIServerPublicIP().DNSName = s.GenerateFQDN(s.APIServerPublicIP().Name) } } diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 0a9625adbf2..2f47164689a 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -146,23 +146,26 @@ func (m *MachineScope) InboundNatSpecs() []azure.InboundNatSpec { // NICSpecs returns the network interface specs. func (m *MachineScope) NICSpecs() []azure.NICSpec { spec := azure.NICSpec{ - Name: azure.GenerateNICName(m.Name()), - MachineName: m.Name(), - VNetName: m.Vnet().Name, - VNetResourceGroup: m.Vnet().ResourceGroup, - SubnetName: m.Subnet().Name, - VMSize: m.AzureMachine.Spec.VMSize, - AcceleratedNetworking: m.AzureMachine.Spec.AcceleratedNetworking, - IPv6Enabled: m.IsIPv6Enabled(), - EnableIPForwarding: m.AzureMachine.Spec.EnableIPForwarding, + Name: azure.GenerateNICName(m.Name()), + MachineName: m.Name(), + VNetName: m.Vnet().Name, + VNetResourceGroup: m.Vnet().ResourceGroup, + SubnetName: m.Subnet().Name, + VMSize: m.AzureMachine.Spec.VMSize, + AcceleratedNetworking: m.AzureMachine.Spec.AcceleratedNetworking, + IPv6Enabled: m.IsIPv6Enabled(), + EnableIPForwarding: m.AzureMachine.Spec.EnableIPForwarding, + PublicLBName: m.OutboundLBName(m.Role()), + PublicLBAddressPoolName: m.OutboundPoolName(m.OutboundLBName(m.Role())), } if m.Role() == infrav1.ControlPlane { - spec.LBName = m.APIServerLBName() - spec.LBBackendAddressPoolName = azure.GenerateBackendAddressPoolName(m.APIServerLBName()) - spec.LBNATRuleName = m.Name() - } else if m.Role() == infrav1.Node { - spec.LBName = m.NodeOutboundLBName() - spec.LBBackendAddressPoolName = azure.GenerateOutboundBackendAddressPoolName(m.NodeOutboundLBName()) + if m.IsAPIServerPrivate() { + spec.InternalLBName = m.APIServerLBName() + spec.InternalLBAddressPoolName = m.APIServerLBPoolName(m.APIServerLBName()) + } else { + spec.PublicLBNATRuleName = m.Name() + spec.PublicLBAddressPoolName = m.APIServerLBPoolName(m.APIServerLBName()) + } } specs := []azure.NICSpec{spec} if m.AzureMachine.Spec.AllocatePublicIP == true { diff --git a/cloud/scope/machinepool.go b/cloud/scope/machinepool.go index 144685744d3..2ebe863da7b 100644 --- a/cloud/scope/machinepool.go +++ b/cloud/scope/machinepool.go @@ -112,8 +112,8 @@ func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec { SubnetName: m.NodeSubnet().Name, VNetName: m.Vnet().Name, VNetResourceGroup: m.Vnet().ResourceGroup, - PublicLBName: m.NodeOutboundLBName(), - PublicLBAddressPoolName: azure.GenerateOutboundBackendAddressPoolName(m.NodeOutboundLBName()), + PublicLBName: m.OutboundLBName(infrav1.Node), + PublicLBAddressPoolName: azure.GenerateOutboundBackendAddressPoolName(m.OutboundLBName(infrav1.Node)), AcceleratedNetworking: m.AzureMachinePool.Spec.Template.AcceleratedNetworking, Identity: m.AzureMachinePool.Spec.Identity, UserAssignedIdentities: m.AzureMachinePool.Spec.UserAssignedIdentities, diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index e90b2c63f8b..5efd66db984 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -211,3 +211,30 @@ func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool { func (s *ManagedControlPlaneScope) IsVnetManaged() bool { return true } + +// APIServerLBName returns the API Server LB name. +func (s *ManagedControlPlaneScope) APIServerLBName() string { + return "" // does not apply for AKS +} + +// APIServerLBPoolName returns the API Server LB backend pool name. +func (s *ManagedControlPlaneScope) APIServerLBPoolName(loadBalancerName string) string { + return "" // does not apply for AKS +} + +// IsAPIServerPrivate returns true if the API Server LB is of type Internal. +// Currently always false as managed control planes do not currently implement private clusters. +func (s *ManagedControlPlaneScope) IsAPIServerPrivate() bool { + return false +} + +// OutboundLBName returns the name of the outbound LB. +// Note: for managed clusters, the outbound LB lifecycle is not managed. +func (s *ManagedControlPlaneScope) OutboundLBName(_ string) string { + return "kubernetes" +} + +// OutboundPoolName returns the outbound LB backend pool name. +func (s *ManagedControlPlaneScope) OutboundPoolName(_ string) string { + return "aksOutboundBackendPool" // hard-coded in aks +} diff --git a/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go b/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go index fb21e534258..a7ef832b5f9 100644 --- a/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go +++ b/cloud/services/bastionhosts/mocks_bastionhosts/bastionhosts_mock.go @@ -412,18 +412,60 @@ func (mr *MockBastionScopeMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockBastionScope)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockBastionScope) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockBastionScope) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockBastionScopeMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockBastionScopeMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockBastionScope)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockBastionScope)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockBastionScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockBastionScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockBastionScope)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockBastionScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockBastionScopeMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockBastionScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockBastionScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockBastionScopeMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockBastionScope)(nil).OutboundPoolName), arg0) } // BastionSpecs mocks base method. diff --git a/cloud/services/loadbalancers/loadbalancers.go b/cloud/services/loadbalancers/loadbalancers.go index 322c1930e65..3e9ba5acb68 100644 --- a/cloud/services/loadbalancers/loadbalancers.go +++ b/cloud/services/loadbalancers/loadbalancers.go @@ -43,7 +43,7 @@ func (s *Service) Reconcile(ctx context.Context) error { } lb := network.LoadBalancer{ - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Sku: &network.LoadBalancerSku{Name: converters.SKUtoSDK(lbSpec.SKU)}, Location: to.StringPtr(s.Scope.Location()), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ ClusterName: s.Scope.ClusterName(), @@ -115,7 +115,6 @@ func (s *Service) Reconcile(ctx context.Context) error { }, }, } - // TODO: check if this is needed for outbound access if lbSpec.Type == infrav1.Internal { lb.LoadBalancerPropertiesFormat.OutboundRules = nil } diff --git a/cloud/services/loadbalancers/loadbalancers_test.go b/cloud/services/loadbalancers/loadbalancers_test.go index 3eca4c79cdc..33fa398717a 100644 --- a/cloud/services/loadbalancers/loadbalancers_test.go +++ b/cloud/services/loadbalancers/loadbalancers_test.go @@ -70,6 +70,7 @@ func TestReconcileLoadBalancer(t *testing.T) { Name: "my-publiclb", Role: infrav1.APIServerRole, Type: infrav1.Public, + SKU: infrav1.SKUStandard, SubnetName: "my-cp-subnet", BackendPoolName: "my-publiclb-backendPool", FrontendIPConfigs: []infrav1.FrontendIP{ @@ -175,6 +176,7 @@ func TestReconcileLoadBalancer(t *testing.T) { Name: "my-private-lb", Role: infrav1.APIServerRole, Type: infrav1.Internal, + SKU: infrav1.SKUStandard, SubnetName: "my-cp-subnet", BackendPoolName: "my-private-lb-backendPool", FrontendIPConfigs: []infrav1.FrontendIP{ @@ -270,6 +272,7 @@ func TestReconcileLoadBalancer(t *testing.T) { Name: "cluster-name", Role: infrav1.NodeOutboundRole, Type: infrav1.Public, + SKU: infrav1.SKUStandard, BackendPoolName: "cluster-name-outboundBackendPool", FrontendIPConfigs: []infrav1.FrontendIP{ { diff --git a/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go b/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go index d8029e1bda3..e4eb3afb4a4 100644 --- a/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go +++ b/cloud/services/loadbalancers/mock_loadbalancers/loadbalancers_mock.go @@ -412,18 +412,60 @@ func (mr *MockLBScopeMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockLBScope)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockLBScope) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockLBScope) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockLBScopeMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockLBScopeMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockLBScope)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockLBScope)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockLBScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockLBScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockLBScope)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockLBScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockLBScopeMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockLBScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockLBScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockLBScopeMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockLBScope)(nil).OutboundPoolName), arg0) } // LBSpecs mocks base method. diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index ac0d893e765..85825eaffd6 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -56,22 +56,29 @@ func (s *Service) Reconcile(ctx context.Context) error { nicConfig.PrivateIPAddress = to.StringPtr(nicSpec.StaticIPAddress) } - if nicSpec.LBName != "" { - if nicSpec.LBBackendAddressPoolName != "" { - nicConfig.LoadBalancerBackendAddressPools = &[]network.BackendAddressPool{ - { - ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.LBName, nicSpec.LBBackendAddressPoolName)), - }, - } + backendAddressPools := []network.BackendAddressPool{} + if nicSpec.PublicLBName != "" { + if nicSpec.PublicLBAddressPoolName != "" { + backendAddressPools = append(backendAddressPools, + network.BackendAddressPool{ + ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.PublicLBName, nicSpec.PublicLBAddressPoolName)), + }) } - if nicSpec.LBNATRuleName != "" { + if nicSpec.PublicLBNATRuleName != "" { nicConfig.LoadBalancerInboundNatRules = &[]network.InboundNatRule{ { - ID: to.StringPtr(azure.NATRuleID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.LBName, nicSpec.LBNATRuleName)), + ID: to.StringPtr(azure.NATRuleID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.PublicLBName, nicSpec.PublicLBNATRuleName)), }, } } } + if nicSpec.InternalLBName != "" && nicSpec.InternalLBAddressPoolName != "" { + backendAddressPools = append(backendAddressPools, + network.BackendAddressPool{ + ID: to.StringPtr(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), nicSpec.InternalLBName, nicSpec.InternalLBAddressPoolName)), + }) + } + nicConfig.LoadBalancerBackendAddressPools = &backendAddressPools if nicSpec.PublicIPName != "" { nicConfig.PublicIPAddress = &network.PublicIPAddress{ diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index 733037daf05..0ec6717c4eb 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -19,9 +19,7 @@ package networkinterfaces import ( "context" "fmt" - "github.com/google/go-cmp/cmp" "net/http" - "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" "testing" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" @@ -32,9 +30,11 @@ import ( "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" + "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" "k8s.io/klog/klogr" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/networkinterfaces/mock_networkinterfaces" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" ) func TestReconcileNetworkInterface(t *testing.T) { @@ -84,7 +84,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - LBName: "my-public-lb", + PublicLBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, }, @@ -105,16 +105,16 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - LBName: "my-public-lb", - LBBackendAddressPoolName: "cluster-name-outboundBackendPool", - StaticIPAddress: "fake.static.ip", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + PublicLBName: "my-public-lb", + PublicLBAddressPoolName: "cluster-name-outboundBackendPool", + StaticIPAddress: "fake.static.ip", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -150,15 +150,15 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - LBName: "my-public-lb", - LBBackendAddressPoolName: "cluster-name-outboundBackendPool", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + PublicLBName: "my-public-lb", + PublicLBAddressPoolName: "cluster-name-outboundBackendPool", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -193,16 +193,18 @@ func TestReconcileNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - MachineName: "azure-test1", - SubnetName: "my-subnet", - VNetName: "my-vnet", - VNetResourceGroup: "my-rg", - LBName: "my-public-lb", - LBBackendAddressPoolName: "my-public-lb-backendPool", - LBNATRuleName: "azure-test1", - VMSize: "Standard_D2v2", - AcceleratedNetworking: nil, + Name: "my-net-interface", + MachineName: "azure-test1", + SubnetName: "my-subnet", + VNetName: "my-vnet", + VNetResourceGroup: "my-rg", + PublicLBName: "my-public-lb", + PublicLBAddressPoolName: "my-public-lb-backendPool", + PublicLBNATRuleName: "azure-test1", + InternalLBName: "my-internal-lb", + InternalLBAddressPoolName: "my-internal-lb-backendPool", + VMSize: "Standard_D2v2", + AcceleratedNetworking: nil, }, }) s.SubscriptionID().AnyTimes().Return("123") @@ -225,7 +227,7 @@ func TestReconcileNetworkInterface(t *testing.T) { LoadBalancerInboundNatRules: &[]network.InboundNatRule{{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-public-lb/inboundNatRules/azure-test1")}}, LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{ {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-public-lb/backendAddressPools/my-public-lb-backendPool")}, - }, + {ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-internal-lb/backendAddressPools/my-internal-lb-backendPool")}}, }, }, }, @@ -269,7 +271,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - LBName: "my-public-lb", + PublicLBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, }, @@ -289,8 +291,9 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, }, }, }, @@ -310,7 +313,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", VNetName: "my-vnet", VNetResourceGroup: "my-rg", - LBName: "my-public-lb", + PublicLBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: to.BoolPtr(false), }, @@ -331,8 +334,9 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, }, }, }, @@ -352,7 +356,7 @@ func TestReconcileNetworkInterface(t *testing.T) { VNetName: "my-vnet", IPv6Enabled: true, VNetResourceGroup: "my-rg", - LBName: "my-public-lb", + PublicLBName: "my-public-lb", VMSize: "Standard_D2v2", AcceleratedNetworking: nil, EnableIPForwarding: true, @@ -374,8 +378,9 @@ func TestReconcileNetworkInterface(t *testing.T) { { Name: to.StringPtr("pipConfig"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, - PrivateIPAllocationMethod: network.Dynamic, + Subnet: &network.Subnet{ID: to.StringPtr("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/my-subnet")}, + PrivateIPAllocationMethod: network.Dynamic, + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, }, }, { @@ -457,9 +462,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - LBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + PublicLBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -473,9 +478,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - LBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + PublicLBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) @@ -490,9 +495,9 @@ func TestDeleteNetworkInterface(t *testing.T) { expect: func(s *mock_networkinterfaces.MockNICScopeMockRecorder, m *mock_networkinterfaces.MockClientMockRecorder) { s.NICSpecs().Return([]azure.NICSpec{ { - Name: "my-net-interface", - LBName: "my-public-lb", - MachineName: "azure-test1", + Name: "my-net-interface", + PublicLBName: "my-public-lb", + MachineName: "azure-test1", }, }) s.ResourceGroup().AnyTimes().Return("my-rg") diff --git a/cloud/services/routetables/mock_routetables/routetables_mock.go b/cloud/services/routetables/mock_routetables/routetables_mock.go index cbaf2aa1156..40d2d4fa3e6 100644 --- a/cloud/services/routetables/mock_routetables/routetables_mock.go +++ b/cloud/services/routetables/mock_routetables/routetables_mock.go @@ -412,18 +412,60 @@ func (mr *MockRouteTableScopeMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockRouteTableScope)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockRouteTableScope) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockRouteTableScope) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockRouteTableScopeMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockRouteTableScopeMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockRouteTableScope)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockRouteTableScope)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockRouteTableScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockRouteTableScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockRouteTableScope)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockRouteTableScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockRouteTableScopeMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockRouteTableScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockRouteTableScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockRouteTableScopeMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockRouteTableScope)(nil).OutboundPoolName), arg0) } // RouteTableSpecs mocks base method. diff --git a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go index 8502d2dd6c9..43a9ed9b263 100644 --- a/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go +++ b/cloud/services/securitygroups/mock_securitygroups/securitygroups_mock.go @@ -412,18 +412,60 @@ func (mr *MockNSGScopeMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockNSGScope)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockNSGScope) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockNSGScope) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockNSGScopeMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockNSGScopeMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockNSGScope)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockNSGScope)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockNSGScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockNSGScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockNSGScope)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockNSGScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockNSGScopeMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockNSGScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockNSGScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockNSGScopeMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockNSGScope)(nil).OutboundPoolName), arg0) } // NSGSpecs mocks base method. diff --git a/cloud/services/subnets/mock_subnets/subnets_mock.go b/cloud/services/subnets/mock_subnets/subnets_mock.go index ff222ac3b2d..cec7544a520 100644 --- a/cloud/services/subnets/mock_subnets/subnets_mock.go +++ b/cloud/services/subnets/mock_subnets/subnets_mock.go @@ -412,18 +412,60 @@ func (mr *MockSubnetScopeMockRecorder) APIServerLBName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBName", reflect.TypeOf((*MockSubnetScope)(nil).APIServerLBName)) } -// NodeOutboundLBName mocks base method. -func (m *MockSubnetScope) NodeOutboundLBName() string { +// APIServerLBPoolName mocks base method. +func (m *MockSubnetScope) APIServerLBPoolName(arg0 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NodeOutboundLBName") + ret := m.ctrl.Call(m, "APIServerLBPoolName", arg0) ret0, _ := ret[0].(string) return ret0 } -// NodeOutboundLBName indicates an expected call of NodeOutboundLBName. -func (mr *MockSubnetScopeMockRecorder) NodeOutboundLBName() *gomock.Call { +// APIServerLBPoolName indicates an expected call of APIServerLBPoolName. +func (mr *MockSubnetScopeMockRecorder) APIServerLBPoolName(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeOutboundLBName", reflect.TypeOf((*MockSubnetScope)(nil).NodeOutboundLBName)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "APIServerLBPoolName", reflect.TypeOf((*MockSubnetScope)(nil).APIServerLBPoolName), arg0) +} + +// IsAPIServerPrivate mocks base method. +func (m *MockSubnetScope) IsAPIServerPrivate() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAPIServerPrivate") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAPIServerPrivate indicates an expected call of IsAPIServerPrivate. +func (mr *MockSubnetScopeMockRecorder) IsAPIServerPrivate() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAPIServerPrivate", reflect.TypeOf((*MockSubnetScope)(nil).IsAPIServerPrivate)) +} + +// OutboundLBName mocks base method. +func (m *MockSubnetScope) OutboundLBName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundLBName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundLBName indicates an expected call of OutboundLBName. +func (mr *MockSubnetScopeMockRecorder) OutboundLBName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundLBName", reflect.TypeOf((*MockSubnetScope)(nil).OutboundLBName), arg0) +} + +// OutboundPoolName mocks base method. +func (m *MockSubnetScope) OutboundPoolName(arg0 string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OutboundPoolName", arg0) + ret0, _ := ret[0].(string) + return ret0 +} + +// OutboundPoolName indicates an expected call of OutboundPoolName. +func (mr *MockSubnetScopeMockRecorder) OutboundPoolName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OutboundPoolName", reflect.TypeOf((*MockSubnetScope)(nil).OutboundPoolName), arg0) } // SubnetSpecs mocks base method. diff --git a/cloud/types.go b/cloud/types.go index df1c8cdc8ed..f15e2e124d5 100644 --- a/cloud/types.go +++ b/cloud/types.go @@ -29,20 +29,22 @@ type PublicIPSpec struct { // NICSpec defines the specification for a Network Interface. type NICSpec struct { - Name string - MachineName string - SubnetName string - VNetName string - VNetResourceGroup string - StaticIPAddress string - LBName string - LBBackendAddressPoolName string - LBNATRuleName string - PublicIPName string - VMSize string - AcceleratedNetworking *bool - IPv6Enabled bool - EnableIPForwarding bool + Name string + MachineName string + SubnetName string + VNetName string + VNetResourceGroup string + StaticIPAddress string + PublicLBName string + PublicLBAddressPoolName string + PublicLBNATRuleName string + InternalLBName string + InternalLBAddressPoolName string + PublicIPName string + VMSize string + AcceleratedNetworking *bool + IPv6Enabled bool + EnableIPForwarding bool } // DiskSpec defines the specification for a Disk. @@ -55,6 +57,7 @@ type LBSpec struct { Name string Role string Type infrav1.LBType + SKU infrav1.SKU SubnetName string BackendPoolName string FrontendIPConfigs []infrav1.FrontendIP From d165cc7c38f64a02e46cb101ee43401fb46c325e Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Wed, 21 Oct 2020 09:51:28 -0700 Subject: [PATCH 6/7] Add iptables workaround for hairpin routing kubeadm init issue Update azure_test.go --- templates/cluster-template-private.yaml | 17 +- templates/cluster-template.yaml | 199 ++++++++++++++++++ templates/flavors/private/kustomization.yaml | 2 +- .../private/patches/ilb-iptables-cmd.yaml | 10 + .../flavors/private/patches/private-lb.yaml | 15 +- .../test/cluster-template-prow-private.yaml | 5 +- test/e2e/azure_privatecluster.go | 3 +- test/e2e/azure_test.go | 1 + 8 files changed, 221 insertions(+), 31 deletions(-) create mode 100644 templates/cluster-template.yaml create mode 100644 templates/flavors/private/patches/ilb-iptables-cmd.yaml diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 8f994ee39c1..df9817fe915 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -28,22 +28,10 @@ spec: location: ${AZURE_LOCATION} networkSpec: apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-internal-lb-frontend - privateIP: 10.128.0.100 name: ${CLUSTER_NAME}-internal-lb type: Internal - subnets: - - cidrBlocks: - - ${AZURE_CP_SUBNET_CIDR} - name: private-cp-subnet - role: control-plane - - cidrBlocks: - - ${AZURE_NODE_SUBNET_CIDR} - name: private-node-subnet - role: node vnet: - name: ${AZURE_VNET_NAME} + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} --- @@ -123,6 +111,9 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + preKubeadmCommands: + - if [ -f /tmp/kubeadm.yaml ]; then iptables -t nat -A OUTPUT -p all -d ${AZURE_INTERNAL_LB_IP} + -j DNAT --to-destination 127.0.0.1; fi useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml new file mode 100644 index 00000000000..ab2bf7c478e --- /dev/null +++ b/templates/cluster-template.yaml @@ -0,0 +1,199 @@ +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + location: ${AZURE_LOCATION} + networkSpec: + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + cluster-name: ${CLUSTER_NAME} + extraVolumes: + - hostPath: /etc/kubernetes/azure.json + mountPath: /etc/kubernetes/azure.json + name: cloud-config + readOnly: true + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + useExperimentalRetryJoin: true + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1alpha3 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + location: ${AZURE_LOCATION} + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-config: /etc/kubernetes/azure.json + cloud-provider: azure + name: '{{ ds.meta_data["local_hostname"] }}' + useExperimentalRetryJoin: true diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml index f154854590a..b96f9b2e7b3 100644 --- a/templates/flavors/private/kustomization.yaml +++ b/templates/flavors/private/kustomization.yaml @@ -3,4 +3,4 @@ resources: - ../default patchesStrategicMerge: - patches/private-lb.yaml - + - patches/ilb-iptables-cmd.yaml diff --git a/templates/flavors/private/patches/ilb-iptables-cmd.yaml b/templates/flavors/private/patches/ilb-iptables-cmd.yaml new file mode 100644 index 00000000000..882d5c70218 --- /dev/null +++ b/templates/flavors/private/patches/ilb-iptables-cmd.yaml @@ -0,0 +1,10 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1alpha3 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + kubeadmConfigSpec: + preKubeadmCommands: + # this is a workaround for the hairpin routing problem with internal load balancers + # during kubeadm init where 10.128.0.100 is the internal LB IP. + - if [ -f /tmp/kubeadm.yaml ]; then iptables -t nat -A OUTPUT -p all -d ${AZURE_INTERNAL_LB_IP} -j DNAT --to-destination 127.0.0.1; fi diff --git a/templates/flavors/private/patches/private-lb.yaml b/templates/flavors/private/patches/private-lb.yaml index ad558c8d8da..cb1e27c1b70 100644 --- a/templates/flavors/private/patches/private-lb.yaml +++ b/templates/flavors/private/patches/private-lb.yaml @@ -4,20 +4,7 @@ metadata: name: ${CLUSTER_NAME} spec: networkSpec: - vnet: - name: ${AZURE_VNET_NAME} - subnets: - - name: private-cp-subnet - role: control-plane - cidrBlocks: - - ${AZURE_CP_SUBNET_CIDR} - - name: private-node-subnet - role: node - cidrBlocks: - - ${AZURE_NODE_SUBNET_CIDR} apiServerLB: name: ${CLUSTER_NAME}-internal-lb type: Internal - frontendIPs: - - name: ${CLUSTER_NAME}-internal-lb-frontend - privateIP: 10.128.0.100 \ No newline at end of file + diff --git a/templates/test/cluster-template-prow-private.yaml b/templates/test/cluster-template-prow-private.yaml index 2dc26d00665..8600300a685 100644 --- a/templates/test/cluster-template-prow-private.yaml +++ b/templates/test/cluster-template-prow-private.yaml @@ -33,7 +33,7 @@ spec: apiServerLB: frontendIPs: - name: ${CLUSTER_NAME}-internal-lb-frontend - privateIP: 10.128.0.100 + privateIP: ${AZURE_INTERNAL_LB_IP} name: ${CLUSTER_NAME}-internal-lb type: Internal subnets: @@ -126,6 +126,9 @@ spec: mounts: - - LABEL=etcd_disk - /var/lib/etcddisk + preKubeadmCommands: + - if [ -f /tmp/kubeadm.yaml ]; then iptables -t nat -A OUTPUT -p all -d ${AZURE_INTERNAL_LB_IP} + -j DNAT --to-destination 127.0.0.1; fi useExperimentalRetryJoin: true replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${KUBERNETES_VERSION} diff --git a/test/e2e/azure_privatecluster.go b/test/e2e/azure_privatecluster.go index 715363024c1..6046cbfc62f 100644 --- a/test/e2e/azure_privatecluster.go +++ b/test/e2e/azure_privatecluster.go @@ -94,8 +94,7 @@ func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivat By("Creating a private workload cluster") clusterName = fmt.Sprintf("capz-e2e-%s", util.RandomString(6)) - Expect(os.Setenv(AzureResourceGroup, input.ClusterName)).NotTo(HaveOccurred()) - Expect(os.Setenv(AzureVNetName, fmt.Sprintf("%s-vnet", input.ClusterName))).NotTo(HaveOccurred()) + Expect(os.Setenv(AzureInternalLBIP, "10.128.0.100")).NotTo(HaveOccurred()) result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: publicClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 9218fc83b85..69957ac00ae 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -123,6 +123,7 @@ var _ = Describe("Workload cluster creation", func() { Context("Creating a private cluster from the management cluster", func() { AzurePrivateClusterSpec(ctx, func() AzurePrivateClusterSpecInput { return AzurePrivateClusterSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, Namespace: namespace, ClusterName: clusterName, ClusterctlConfigPath: clusterctlConfigPath, From fd5c6ad88d9cf6175e8991f5a03b88a2cd9c8033 Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 29 Oct 2020 13:36:36 -0700 Subject: [PATCH 7/7] Update managedcontrolplane.go with added NetworkDescriber functions --- cloud/scope/managedcontrolplane.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud/scope/managedcontrolplane.go b/cloud/scope/managedcontrolplane.go index 5efd66db984..74f554183bb 100644 --- a/cloud/scope/managedcontrolplane.go +++ b/cloud/scope/managedcontrolplane.go @@ -203,6 +203,7 @@ func (s *ManagedControlPlaneScope) ControlPlaneSubnet() *infrav1.SubnetSpec { } // IsIPv6Enabled returns true if a cluster is ipv6 enabled. +// Currently always false as managed control planes do not currently implement ipv6. func (s *ManagedControlPlaneScope) IsIPv6Enabled() bool { return false }