From 012f0da3bd2f1a962f9b832522dd0d12f3495ae1 Mon Sep 17 00:00:00 2001 From: "Kistner, Dominic" Date: Tue, 19 Jan 2021 08:46:18 +0100 Subject: [PATCH] =?UTF-8?q?NatGateway=20Integration=20=E2=80=93=20Step=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../azure-infra/templates/_helpers.tpl | 39 ++++++++ charts/internal/azure-infra/templates/main.tf | 81 ++++++----------- charts/internal/azure-infra/values.yaml | 13 ++- docs/usage-as-end-user.md | 31 ++++--- hack/api-reference/api.md | 79 +++++++++++++++- pkg/apis/azure/types_infrastructure.go | 30 +++++-- .../azure/v1alpha1/types_infrastructure.go | 31 +++++-- .../azure/v1alpha1/zz_generated.conversion.go | 38 ++++++++ .../azure/v1alpha1/zz_generated.deepcopy.go | 26 ++++++ pkg/apis/azure/validation/infrastructure.go | 80 +++++++++++------ .../azure/validation/infrastructure_test.go | 89 +++++++++++++++++-- pkg/apis/azure/zz_generated.deepcopy.go | 26 ++++++ pkg/internal/infrastructure/terraform.go | 65 +++++++++----- pkg/internal/infrastructure/terraform_test.go | 43 ++++++++- 14 files changed, 529 insertions(+), 142 deletions(-) create mode 100644 charts/internal/azure-infra/templates/_helpers.tpl diff --git a/charts/internal/azure-infra/templates/_helpers.tpl b/charts/internal/azure-infra/templates/_helpers.tpl new file mode 100644 index 000000000..f6af52886 --- /dev/null +++ b/charts/internal/azure-infra/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{- define "resource-group-reference" -}} +{{- if .Values.create.resourceGroup -}} +azurerm_resource_group.rg.name +{{- else -}} +data.azurerm_resource_group.rg.name +{{- end}} +{{- end -}} + +{{- define "natgateway-managed-ip" -}} +# Gardener managed public IP to be attached to the NatGateway. +resource "azurerm_public_ip" "natip" { + name = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip" + location = "{{ required "azure.region is required" .Values.azure.region }}" + resource_group_name = {{ template "resource-group-reference" . }} + allocation_method = "Static" + sku = "Standard" + {{ if .Values.natGateway -}}{{ if hasKey .Values.natGateway "zone" -}} + zones = [{{ .Values.natGateway.zone | quote }}] + {{- end }}{{- end }} +} +resource "azurerm_nat_gateway_public_ip_association" "natip-association" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.natip.id +} +{{- end -}} + +{{- define "natgateway-user-provided-public-ips" -}} +# User provided public IPs to be attached to the NatGateway. +{{ range $index, $ip := .Values.natGateway.ipAddresses -}} +data "azurerm_public_ip" "natip-user-provided-{{ $index }}" { + name = "{{ $ip.name }}" + resource_group_name = "{{ $ip.resourceGroup }}" +} +resource "azurerm_nat_gateway_public_ip_association" "natip-user-provided-{{ $index }}-association" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = data.azurerm_public_ip.natip-user-provided-{{ $index }}.id +} +{{ end }} +{{- end -}} diff --git a/charts/internal/azure-infra/templates/main.tf b/charts/internal/azure-infra/templates/main.tf index 119be7da8..e9d5e093b 100644 --- a/charts/internal/azure-infra/templates/main.tf +++ b/charts/internal/azure-infra/templates/main.tf @@ -7,6 +7,9 @@ provider "azurerm" { features {} } +#=============================================== +#= Resource Group +#=============================================== {{ if .Values.create.resourceGroup -}} resource "azurerm_resource_group" "rg" { name = "{{ required "resourceGroup.name is required" .Values.resourceGroup.name }}" @@ -22,14 +25,11 @@ data "azurerm_resource_group" "rg" { #= VNet, Subnets, Route Table, Security Groups #=============================================== +# VNet {{ if .Values.create.vnet -}} resource "azurerm_virtual_network" "vnet" { name = "{{ required "resourceGroup.vnet.name is required" .Values.resourceGroup.vnet.name }}" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end}} + resource_group_name = {{ template "resource-group-reference" . }} location = "{{ required "azure.region is required" .Values.azure.region }}" address_space = ["{{ required "resourceGroup.vnet.cidr is required" .Values.resourceGroup.vnet.cidr }}"] } @@ -40,6 +40,7 @@ data "azurerm_virtual_network" "vnet" { } {{- end }} +# Subnet resource "azurerm_subnet" "workers" { name = "{{ required "clusterName is required" .Values.clusterName }}-nodes" {{ if .Values.create.vnet -}} @@ -53,31 +54,23 @@ resource "azurerm_subnet" "workers" { service_endpoints = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}] } +# RouteTable resource "azurerm_route_table" "workers" { name = "worker_route_table" location = "{{ required "azure.region is required" .Values.azure.region }}" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end}} + resource_group_name = {{ template "resource-group-reference" . }} } - resource "azurerm_subnet_route_table_association" "workers-rt-subnet-association" { subnet_id = azurerm_subnet.workers.id route_table_id = azurerm_route_table.workers.id } +# SecurityGroup resource "azurerm_network_security_group" "workers" { name = "{{ required "clusterName is required" .Values.clusterName }}-workers" location = "{{ required "azure.region is required" .Values.azure.region }}" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end}} + resource_group_name = {{ template "resource-group-reference" . }} } - resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet-association" { subnet_id = azurerm_subnet.workers.id network_security_group_id = azurerm_network_security_group.workers.id @@ -88,48 +81,38 @@ resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet #= NAT Gateway #=============================================== -resource "azurerm_public_ip" "natip" { - name = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip" - location = "{{ required "azure.region is required" .Values.azure.region }}" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end }} - allocation_method = "Static" - sku = "Standard" -} - resource "azurerm_nat_gateway" "nat" { name = "{{ required "clusterName is required" .Values.clusterName }}-nat-gateway" location = "{{ required "azure.region is required" .Values.azure.region }}" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end }} + resource_group_name = {{ template "resource-group-reference" . }} sku_name = "Standard" {{ if .Values.natGateway -}} - {{ if .Values.natGateway.idleConnectionTimeoutMinutes -}} + {{ if hasKey .Values.natGateway "idleConnectionTimeoutMinutes" -}} idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }} {{- end }} - - # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed. + {{ if hasKey .Values.natGateway "zone" -}} + zones = [{{ .Values.natGateway.zone | quote }}] + {{- end }} {{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}} + # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed. public_ip_address_ids = [] {{- end }} {{- end }} } - -resource "azurerm_nat_gateway_public_ip_association" "natip-association" { - nat_gateway_id = azurerm_nat_gateway.nat.id - public_ip_address_id = azurerm_public_ip.natip.id -} - resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" { subnet_id = azurerm_subnet.workers.id nat_gateway_id = azurerm_nat_gateway.nat.id } + +{{ if .Values.natGateway -}} +{{ if and (hasKey .Values.natGateway "ipAddresses") (hasKey .Values.natGateway "zone") -}} +{{ template "natgateway-user-provided-public-ips" . }} +{{- else -}} +{{ template "natgateway-managed-ip" . }} +{{- end }} +{{- else -}} +{{ template "natgateway-managed-ip" . }} +{{- end }} {{- end }} {{ if .Values.identity -}} @@ -150,12 +133,8 @@ data "azurerm_user_assigned_identity" "identity" { resource "azurerm_availability_set" "workers" { name = "{{ required "clusterName is required" .Values.clusterName }}-avset-workers" - {{ if .Values.create.resourceGroup -}} - resource_group_name = azurerm_resource_group.rg.name - {{- else -}} - resource_group_name = data.azurerm_resource_group.rg.name - {{- end}} location = "{{ required "azure.region is required" .Values.azure.region }}" + resource_group_name = {{ template "resource-group-reference" . }} platform_update_domain_count = "{{ required "azure.countUpdateDomains is required" .Values.azure.countUpdateDomains }}" platform_fault_domain_count = "{{ required "azure.countFaultDomains is required" .Values.azure.countFaultDomains }}" managed = true @@ -167,11 +146,7 @@ resource "azurerm_availability_set" "workers" { #=============================================== output "{{ .Values.outputKeys.resourceGroupName }}" { -{{- if .Values.create.resourceGroup }} - value = azurerm_resource_group.rg.name -{{- else -}} - value = data.azurerm_resource_group.rg.name -{{- end}} + value = {{ template "resource-group-reference" . }} } {{ if .Values.create.vnet -}} diff --git a/charts/internal/azure-infra/values.yaml b/charts/internal/azure-infra/values.yaml index aa23eebe9..7b936a714 100644 --- a/charts/internal/azure-infra/values.yaml +++ b/charts/internal/azure-infra/values.yaml @@ -15,10 +15,15 @@ create: # name: identity-name # resourceGroup: identity-resource-group -natGateway: - idleConnectionTimeoutMinutes: - # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed. - migrateNatGatewayToIPAssociation: false +# natGateway: + # idleConnectionTimeoutMinutes: 4 + # zone: 1 + # migrateNatGatewayToIPAssociation: false # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed. + # ipAddresses: + # - name: public-ip-0-name + # resourceGroup: public-ip-0-resource-group + # - name: public-ip-1-name + # resourceGroup: public-ip-1-resource-group resourceGroup: name: my-resource-group diff --git a/docs/usage-as-end-user.md b/docs/usage-as-end-user.md index 8335cc1e8..6771953b2 100644 --- a/docs/usage-as-end-user.md +++ b/docs/usage-as-end-user.md @@ -51,6 +51,11 @@ networks: # natGateway: # enabled: false # idleConnectionTimeoutMinutes: 4 + # zone: 1 + # ipAddresses: + # - name: my-public-ip-name + # resourceGroup: my-public-ip-resource-group + # zone: 1 # serviceEndpoints: # - Microsoft.Test zoned: false @@ -62,6 +67,13 @@ zoned: false # acrAccess: true ``` +Currently, it's not yet possible to deploy into existing resource groups, but in the future it will. +The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to. + +Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not. +If you don't use zones then an availability set will be created and only basic load balancers will be used. +Zoned clusters use standard load balancers. + The `networks.vnet` section describes whether you want to create the shoot cluster in an already existing VNet or whether to create a new one: * If `networks.vnet.name` and `networks.vnet.resourceGroup` are given then you have to specify the VNet name and VNet resource group name of the existing VNet that was created by other means (manually, other tooling, ...). @@ -75,18 +87,17 @@ You can freely choose this CIDR and it is your responsibility to properly design In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview). -The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. If NAT Gateway is enabled, the field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers). - -Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not. -If you don't use zones then an availability set will be created and only basic load balancers will be used. -Zoned clusters use standard load balancers. +The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of a Shoot cluster. Here are some key information about the usage of the NatGateway for a Shoot cluster: +- NatGateway usage is optional and can be enabled or disabled via `.networks.natGateway.enabled`. +- If the NatGateway is not used then the egress connections initiated within the Shoot cluster will be nated via the LoadBalancer of the clusters (default Azure behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). +- NatGateway is only available for zonal clusters `.zoned=true`. +- The NatGateway is currently **not** zone redundantly deployed. That mean the NatGateway of a Shoot cluster will always be in just one zone. Those zone can be optionally selected via `networks.natGateway.zone`. +- **Caution:** Modifing the `networks.natGateway.zone` setting require a recreation of the NatGateway and the managed public ip (automatically used if no own public ip is specified, see below). That mean you will most likely get a different public ip for egress connections. +- It is possible to bring own zonal public ip(s) via `networks.natGateway.ipAddresses`. Those public ip(s) need to be in the same zone as the NatGateway (see `networks.natGateway.zone`) and be of SKU `standard`. For each public the `name`, the `resourceGroup` and the `zone` need to be specified. +- The field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers). In the `identity` section you can specify an [Azure user-assigned managed identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview#how-does-the-managed-identities-for-azure-resources-work) which should be attached to all cluster worker machines. With `identity.name` you can specify the name of the identity and with `identity.resourceGroup` you can specify the resource group which contains the identity resource on Azure. The identity need to be created by the user upfront (manually, other tooling, ...). Gardener/Azure Extension will only use the referenced one and won't create an identity. Furthermore the identity have to be in the same subscription as the Shoot cluster. Via the `identity.acrAccess` you can configure the worker machines to use the passed identity for pulling from an [Azure Container Registry (ACR)](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). - -Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster. - -Currently, it's not yet possible to deploy into existing resource groups, but in the future it will. -The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to. +**Caution:** Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster. Apart from the VNet and the worker subnet the Azure extension will also create a dedicated resource group, route tables, security groups, and an availability set (if not using zoned clusters). diff --git a/hack/api-reference/api.md b/hack/api-reference/api.md index e7b87ce75..8edf06c22 100644 --- a/hack/api-reference/api.md +++ b/hack/api-reference/api.md @@ -909,7 +909,7 @@ bool NetworkConfig)

-

NatGatewayConfig contains configuration for the nat gateway and the attached resources.

+

NatGatewayConfig contains configuration for the NAT gateway and the attached resources.

@@ -938,10 +938,33 @@ int32 + + + + + + + +
-(Optional)

IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes.
+zone
+ +int32 + +		 +

Zone specifies the zone in which the NAT gateway should be deployed to.

+
+ipAddresses
+ + +[]PublicIPReference + + +		 +

IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.

+

NetworkConfig @@ -1058,6 +1081,58 @@ VNetStatus +

PublicIPReference +

+

+(Appears on: +NatGatewayConfig) +

+

+

PublicIPReference contains information about a public ip.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ +string + +		 +

Name is the name of the public ip.

+
+resourceGroup
+ +string + +		 +

ResourceGroup is the name of the resource group where the public ip is assigned to.

+
+zone
+ +int32 + +		 +

Zone is the zone in which the public ip is deployed to.

+

Purpose (string alias)

diff --git a/pkg/apis/azure/types_infrastructure.go b/pkg/apis/azure/types_infrastructure.go index 4eecd2fc0..cedf0c277 100644 --- a/pkg/apis/azure/types_infrastructure.go +++ b/pkg/apis/azure/types_infrastructure.go @@ -51,6 +51,28 @@ type NetworkConfig struct { ServiceEndpoints []string } +// NatGatewayConfig contains configuration for the NAT gateway and the attached resources. +type NatGatewayConfig struct { + // Enabled is an indicator if NAT gateway should be deployed. + Enabled bool + // IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes. + IdleConnectionTimeoutMinutes *int32 + // Zone specifies the zone in which the NAT gateway should be deployed to. + Zone *int32 + // IPAddresses is a list of ip addresses which should be assigned to the NAT gateway. + IPAddresses []PublicIPReference +} + +// PublicIPReference contains information about a public ip. +type PublicIPReference struct { + // Name is the name of the public ip. + Name string + // ResourceGroup is the name of the resource group where the public ip is assigned to. + ResourceGroup string + // Zone is the zone in which the public ip is deployed to. + Zone int32 +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // InfrastructureStatus contains information about created infrastructure resources. @@ -149,14 +171,6 @@ type VNetStatus struct { ResourceGroup *string } -// NatGatewayConfig contains configuration for the nat gateway and the attached resources. -type NatGatewayConfig struct { - // Enabled is an indicator if NAT gateway should be deployed. - Enabled bool - // IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes. - IdleConnectionTimeoutMinutes *int32 -} - // IdentityConfig contains configuration for the managed identity. type IdentityConfig struct { // Name is the name of the identity. diff --git a/pkg/apis/azure/v1alpha1/types_infrastructure.go b/pkg/apis/azure/v1alpha1/types_infrastructure.go index 069dce087..d593e18d9 100644 --- a/pkg/apis/azure/v1alpha1/types_infrastructure.go +++ b/pkg/apis/azure/v1alpha1/types_infrastructure.go @@ -57,6 +57,28 @@ type NetworkConfig struct { ServiceEndpoints []string `json:"serviceEndpoints,omitempty"` } +// NatGatewayConfig contains configuration for the NAT gateway and the attached resources. +type NatGatewayConfig struct { + // Enabled is an indicator if NAT gateway should be deployed. + Enabled bool `json:"enabled"` + // IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes. + IdleConnectionTimeoutMinutes *int32 `json:"idleConnectionTimeoutMinutes,omitempty"` + // Zone specifies the zone in which the NAT gateway should be deployed to. + Zone *int32 `json:"zone,omitempty"` + // IPAddresses is a list of ip addresses which should be assigned to the NAT gateway. + IPAddresses []PublicIPReference `json:"ipAddresses,omitempty"` +} + +// PublicIPReference contains information about a public ip. +type PublicIPReference struct { + // Name is the name of the public ip. + Name string `json:"name"` + // ResourceGroup is the name of the resource group where the public ip is assigned to. + ResourceGroup string `json:"resourceGroup"` + // Zone is the zone in which the public ip is deployed to. + Zone int32 `json:"zone"` +} + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // InfrastructureStatus contains information about created infrastructure resources. @@ -165,15 +187,6 @@ type VNetStatus struct { ResourceGroup *string `json:"resourceGroup,omitempty"` } -// NatGatewayConfig contains configuration for the nat gateway and the attached resources. -type NatGatewayConfig struct { - // Enabled is an indicator if NAT gateway should be deployed. - Enabled bool `json:"enabled"` - // IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes. - // +optional - IdleConnectionTimeoutMinutes *int32 `json:"idleConnectionTimeoutMinutes,omitempty"` -} - // IdentityConfig contains configuration for the managed identity. type IdentityConfig struct { // Name is the name of the identity. diff --git a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go index 43bea77a2..c26c39a9c 100644 --- a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go @@ -195,6 +195,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*PublicIPReference)(nil), (*azure.PublicIPReference)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_PublicIPReference_To_azure_PublicIPReference(a.(*PublicIPReference), b.(*azure.PublicIPReference), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*azure.PublicIPReference)(nil), (*PublicIPReference)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_azure_PublicIPReference_To_v1alpha1_PublicIPReference(a.(*azure.PublicIPReference), b.(*PublicIPReference), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ResourceGroup)(nil), (*azure.ResourceGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_ResourceGroup_To_azure_ResourceGroup(a.(*ResourceGroup), b.(*azure.ResourceGroup), scope) }); err != nil { @@ -615,6 +625,8 @@ func Convert_azure_MachineType_To_v1alpha1_MachineType(in *azure.MachineType, ou func autoConvert_v1alpha1_NatGatewayConfig_To_azure_NatGatewayConfig(in *NatGatewayConfig, out *azure.NatGatewayConfig, s conversion.Scope) error { out.Enabled = in.Enabled out.IdleConnectionTimeoutMinutes = (*int32)(unsafe.Pointer(in.IdleConnectionTimeoutMinutes)) + out.Zone = (*int32)(unsafe.Pointer(in.Zone)) + out.IPAddresses = *(*[]azure.PublicIPReference)(unsafe.Pointer(&in.IPAddresses)) return nil } @@ -626,6 +638,8 @@ func Convert_v1alpha1_NatGatewayConfig_To_azure_NatGatewayConfig(in *NatGatewayC func autoConvert_azure_NatGatewayConfig_To_v1alpha1_NatGatewayConfig(in *azure.NatGatewayConfig, out *NatGatewayConfig, s conversion.Scope) error { out.Enabled = in.Enabled out.IdleConnectionTimeoutMinutes = (*int32)(unsafe.Pointer(in.IdleConnectionTimeoutMinutes)) + out.Zone = (*int32)(unsafe.Pointer(in.Zone)) + out.IPAddresses = *(*[]PublicIPReference)(unsafe.Pointer(&in.IPAddresses)) return nil } @@ -690,6 +704,30 @@ func Convert_azure_NetworkStatus_To_v1alpha1_NetworkStatus(in *azure.NetworkStat return autoConvert_azure_NetworkStatus_To_v1alpha1_NetworkStatus(in, out, s) } +func autoConvert_v1alpha1_PublicIPReference_To_azure_PublicIPReference(in *PublicIPReference, out *azure.PublicIPReference, s conversion.Scope) error { + out.Name = in.Name + out.ResourceGroup = in.ResourceGroup + out.Zone = in.Zone + return nil +} + +// Convert_v1alpha1_PublicIPReference_To_azure_PublicIPReference is an autogenerated conversion function. +func Convert_v1alpha1_PublicIPReference_To_azure_PublicIPReference(in *PublicIPReference, out *azure.PublicIPReference, s conversion.Scope) error { + return autoConvert_v1alpha1_PublicIPReference_To_azure_PublicIPReference(in, out, s) +} + +func autoConvert_azure_PublicIPReference_To_v1alpha1_PublicIPReference(in *azure.PublicIPReference, out *PublicIPReference, s conversion.Scope) error { + out.Name = in.Name + out.ResourceGroup = in.ResourceGroup + out.Zone = in.Zone + return nil +} + +// Convert_azure_PublicIPReference_To_v1alpha1_PublicIPReference is an autogenerated conversion function. +func Convert_azure_PublicIPReference_To_v1alpha1_PublicIPReference(in *azure.PublicIPReference, out *PublicIPReference, s conversion.Scope) error { + return autoConvert_azure_PublicIPReference_To_v1alpha1_PublicIPReference(in, out, s) +} + func autoConvert_v1alpha1_ResourceGroup_To_azure_ResourceGroup(in *ResourceGroup, out *azure.ResourceGroup, s conversion.Scope) error { out.Name = in.Name return nil diff --git a/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go index 9a65da906..d1ce7769d 100644 --- a/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/azure/v1alpha1/zz_generated.deepcopy.go @@ -404,6 +404,16 @@ func (in *NatGatewayConfig) DeepCopyInto(out *NatGatewayConfig) { *out = new(int32) **out = **in } + if in.Zone != nil { + in, out := &in.Zone, &out.Zone + *out = new(int32) + **out = **in + } + if in.IPAddresses != nil { + in, out := &in.IPAddresses, &out.IPAddresses + *out = make([]PublicIPReference, len(*in)) + copy(*out, *in) + } return } @@ -466,6 +476,22 @@ func (in *NetworkStatus) DeepCopy() *NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PublicIPReference) DeepCopyInto(out *PublicIPReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublicIPReference. +func (in *PublicIPReference) DeepCopy() *PublicIPReference { + if in == nil { + return nil + } + out := new(PublicIPReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceGroup) DeepCopyInto(out *ResourceGroup) { *out = *in diff --git a/pkg/apis/azure/validation/infrastructure.go b/pkg/apis/azure/validation/infrastructure.go index d23760042..992fcdb48 100644 --- a/pkg/apis/azure/validation/infrastructure.go +++ b/pkg/apis/azure/validation/infrastructure.go @@ -68,23 +68,72 @@ func ValidateInfrastructureConfig(infra *apisazure.InfrastructureConfig, nodesCI workerCIDR := cidrvalidation.NewCIDR(infra.Networks.Workers, networksPath.Child("workers")) allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(workerCIDR)...) allErrs = append(allErrs, cidrvalidation.ValidateCIDRIsCanonical(networksPath.Child("workers"), infra.Networks.Workers)...) + if nodes != nil { + allErrs = append(allErrs, nodes.ValidateSubset(workerCIDR)...) + } - // Validate vnet config allErrs = append(allErrs, validateVnetConfig(infra.Networks.VNet, infra.ResourceGroup, workerCIDR, nodes, pods, services, networksPath.Child("vnet"))...) - - allErrs = append(allErrs, validateNatGatewayConfig(infra, hasVmoAlphaAnnotation, fldPath.Child("networks", "natGateway"))...) + allErrs = append(allErrs, validateNatGatewayConfig(infra.Networks.NatGateway, infra.Zoned, hasVmoAlphaAnnotation, networksPath.Child("natGateway"))...) if infra.Identity != nil && (infra.Identity.Name == "" || infra.Identity.ResourceGroup == "") { allErrs = append(allErrs, field.Invalid(fldPath.Child("identity"), infra.Identity, "specifying an identity requires the name of the identity and the resource group which hosts the identity")) } - if nodes != nil { - allErrs = append(allErrs, nodes.ValidateSubset(workerCIDR)...) + return allErrs +} + +func validateNatGatewayConfig(natGatewayConfig *apisazure.NatGatewayConfig, zoned bool, hasVmoAlphaAnnotation bool, natGatewayPath *field.Path) field.ErrorList { + var allErrs = field.ErrorList{} + + if natGatewayConfig == nil { + return nil + } + + if !natGatewayConfig.Enabled { + if natGatewayConfig.Zone != nil || natGatewayConfig.IdleConnectionTimeoutMinutes != nil || natGatewayConfig.IPAddresses != nil { + return append(allErrs, field.Invalid(natGatewayPath, natGatewayConfig, "NatGateway is disabled but additional NatGateway config is passed")) + } + return nil + } + + // NatGateway cannot be offered for Shoot clusters with a primary AvailabilitySet. + // The NatGateway is not compatible with the Basic SKU Loadbalancers which are + // required to use for *Shoot clusters with AvailabilitySet. + if !zoned && !hasVmoAlphaAnnotation { + return append(allErrs, field.Forbidden(natGatewayPath, "NatGateway is currently only supported for zonal and VMO clusters")) + } + + if natGatewayConfig.IdleConnectionTimeoutMinutes != nil && (*natGatewayConfig.IdleConnectionTimeoutMinutes < natGatewayMinTimeoutInMinutes || *natGatewayConfig.IdleConnectionTimeoutMinutes > natGatewayMaxTimeoutInMinutes) { + allErrs = append(allErrs, field.Invalid(natGatewayPath.Child("idleConnectionTimeoutMinutes"), *natGatewayConfig.IdleConnectionTimeoutMinutes, "idleConnectionTimeoutMinutes values must range between 4 and 120")) + } + + if natGatewayConfig.Zone == nil { + if len(natGatewayConfig.IPAddresses) > 0 { + allErrs = append(allErrs, field.Invalid(natGatewayPath.Child("zone"), *natGatewayConfig, "Public IPs can only be selected for zonal NatGateways")) + } + return allErrs } + allErrs = append(allErrs, validateNatGatewayIPReference(natGatewayConfig.IPAddresses, *natGatewayConfig.Zone, natGatewayPath.Child("ipAddresses"))...) return allErrs } +func validateNatGatewayIPReference(references []apisazure.PublicIPReference, zone int32, fldPath *field.Path) field.ErrorList { + var allErrs = field.ErrorList{} + for i, ref := range references { + if ref.Zone != zone { + allErrs = append(allErrs, field.Invalid(fldPath.Index(i).Child("zone"), ref.Zone, fmt.Sprintf("Public IP can't be used as it is not the same zone as the NatGateway (zone %d)", zone))) + } + if ref.Name == "" { + allErrs = append(allErrs, field.Invalid(fldPath.Index(i).Child("name"), ref.Name, "Empty name for NatGateway public IP is invalid")) + } + if ref.ResourceGroup == "" { + allErrs = append(allErrs, field.Invalid(fldPath.Index(i).Child("resourceGroup"), ref.ResourceGroup, "Empty resourceGroup for NatGateway public IP is invalid")) + } + } + return allErrs +} + func validateVnetConfig(vnetConfig apisazure.VNet, resourceGroupConfig *apisazure.ResourceGroup, workers, nodes, pods, services cidrvalidation.CIDR, vnetConfigPath *field.Path) field.ErrorList { var allErrs = field.ErrorList{} @@ -121,27 +170,6 @@ func validateVnetConfig(vnetConfig apisazure.VNet, resourceGroupConfig *apisazur return allErrs } -func validateNatGatewayConfig(infra *apisazure.InfrastructureConfig, hasVmoAlphaAnnotation bool, natGatewayConfigPath *field.Path) field.ErrorList { - var allErrs = field.ErrorList{} - - if infra.Networks.NatGateway == nil { - return allErrs - } - - // NatGateway can't be offered for Shoot clusters with a primary AvailabilitySet. - // The NatGateway is not compatible with the Basic SKU Loadbalancers which are - // required for Shoot clusters with AvailabilitySet. - if !infra.Zoned && !hasVmoAlphaAnnotation { - return append(allErrs, field.Invalid(natGatewayConfigPath, infra.Networks.NatGateway, "NatGateway is currently only supported for zoned cluster")) - } - - if infra.Networks.NatGateway.IdleConnectionTimeoutMinutes != nil && (*infra.Networks.NatGateway.IdleConnectionTimeoutMinutes < natGatewayMinTimeoutInMinutes || *infra.Networks.NatGateway.IdleConnectionTimeoutMinutes > natGatewayMaxTimeoutInMinutes) { - allErrs = append(allErrs, field.Invalid(natGatewayConfigPath.Child("idleConnectionTimeoutMinutes"), *infra.Networks.NatGateway.IdleConnectionTimeoutMinutes, "idleConnectionTimeoutMinutes values must range between 4 and 120")) - } - - return allErrs -} - // ValidateInfrastructureConfigUpdate validates a InfrastructureConfig object. func ValidateInfrastructureConfigUpdate(oldConfig, newConfig *apisazure.InfrastructureConfig, providerPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/pkg/apis/azure/validation/infrastructure_test.go b/pkg/apis/azure/validation/infrastructure_test.go index 9b1533bdc..6249df157 100644 --- a/pkg/apis/azure/validation/infrastructure_test.go +++ b/pkg/apis/azure/validation/infrastructure_test.go @@ -293,24 +293,101 @@ var _ = Describe("InfrastructureConfig validation", func() { }) Context("NatGateway", func() { - It("should return no errors using a NatGateway for a zoned cluster", func() { + BeforeEach(func() { infrastructureConfig.Zoned = true infrastructureConfig.Networks.NatGateway = &apisazure.NatGatewayConfig{Enabled: true} + }) + + It("should pass as there is no NatGateway config is provided", func() { + infrastructureConfig.Networks.NatGateway = nil Expect(ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath)).To(BeEmpty()) }) - It("should return an error using a NatGateway for a non zoned cluster", func() { - infrastructureConfig.Zoned = false - infrastructureConfig.Networks.NatGateway = &apisazure.NatGatewayConfig{} + It("should pass as the NatGateway is disabled", func() { + infrastructureConfig.Networks.NatGateway.Enabled = false + Expect(ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath)).To(BeEmpty()) + }) + + It("should fail as NatGatway is disabled but additional config for the NatGateway is supplied", func() { + infrastructureConfig.Networks.NatGateway.Enabled = false + infrastructureConfig.Networks.NatGateway.Zone = pointer.Int32Ptr(2) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) - Expect(errorList).To(HaveLen(1)) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.natGateway"), - "Detail": Equal("NatGateway is currently only supported for zoned cluster"), + "Detail": Equal("NatGateway is disabled but additional NatGateway config is passed"), })) }) + It("should fail as NatGatway is enabled but the cluster is not zonal", func() { + infrastructureConfig.Zoned = false + infrastructureConfig.Networks.NatGateway.Enabled = true + + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeForbidden), + "Field": Equal("networks.natGateway"), + "Detail": Equal("NatGateway is currently only supported for zonal and VMO clusters"), + })) + }) + + It("should pass as the NatGateway has a zone", func() { + infrastructureConfig.Networks.NatGateway.Zone = pointer.Int32Ptr(2) + Expect(ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath)).To(BeEmpty()) + }) + + Context("User provided public IP", func() { + BeforeEach(func() { + infrastructureConfig.Networks.NatGateway.Zone = pointer.Int32Ptr(1) + infrastructureConfig.Networks.NatGateway.IPAddresses = []apisazure.PublicIPReference{{ + Name: "public-ip-name", + ResourceGroup: "public-ip-resource-group", + Zone: 1, + }} + }) + + It("should fail as NatGateway has no zone but an external public ip", func() { + infrastructureConfig.Networks.NatGateway.Zone = nil + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("networks.natGateway.zone"), + "Detail": Equal("Public IPs can only be selected for zonal NatGateways"), + })) + }) + + It("should fail as resource is in a different zone as the NatGateway", func() { + infrastructureConfig.Networks.NatGateway.IPAddresses[0].Zone = 2 + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("networks.natGateway.ipAddresses[0].zone"), + "Detail": Equal("Public IP can't be used as it is not the same zone as the NatGateway (zone 1)"), + })) + }) + + It("should fail as name is empty", func() { + infrastructureConfig.Networks.NatGateway.IPAddresses[0].Name = "" + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("networks.natGateway.ipAddresses[0].name"), + "Detail": Equal("Empty name for NatGateway public IP is invalid"), + })) + }) + + It("should fail as resource group is empty", func() { + infrastructureConfig.Networks.NatGateway.IPAddresses[0].ResourceGroup = "" + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, hasVmoAlphaAnnotation, providerPath) + Expect(errorList).To(ConsistOfFields(Fields{ + "Type": Equal(field.ErrorTypeInvalid), + "Field": Equal("networks.natGateway.ipAddresses[0].resourceGroup"), + "Detail": Equal("Empty resourceGroup for NatGateway public IP is invalid"), + })) + }) + }) + Context("IdleConnectionTimeoutMinutes", func() { It("should return an error when specifying lower than minimum values", func() { var timeoutValue int32 = 0 diff --git a/pkg/apis/azure/zz_generated.deepcopy.go b/pkg/apis/azure/zz_generated.deepcopy.go index 3ff7ef7fb..92aca501b 100644 --- a/pkg/apis/azure/zz_generated.deepcopy.go +++ b/pkg/apis/azure/zz_generated.deepcopy.go @@ -404,6 +404,16 @@ func (in *NatGatewayConfig) DeepCopyInto(out *NatGatewayConfig) { *out = new(int32) **out = **in } + if in.Zone != nil { + in, out := &in.Zone, &out.Zone + *out = new(int32) + **out = **in + } + if in.IPAddresses != nil { + in, out := &in.IPAddresses, &out.IPAddresses + *out = make([]PublicIPReference, len(*in)) + copy(*out, *in) + } return } @@ -466,6 +476,22 @@ func (in *NetworkStatus) DeepCopy() *NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PublicIPReference) DeepCopyInto(out *PublicIPReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PublicIPReference. +func (in *PublicIPReference) DeepCopy() *PublicIPReference { + if in == nil { + return nil + } + out := new(PublicIPReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceGroup) DeepCopyInto(out *ResourceGroup) { *out = *in diff --git a/pkg/internal/infrastructure/terraform.go b/pkg/internal/infrastructure/terraform.go index 025298a1f..2324cd194 100644 --- a/pkg/internal/infrastructure/terraform.go +++ b/pkg/internal/infrastructure/terraform.go @@ -71,6 +71,26 @@ var StatusTypeMeta = metav1.TypeMeta{ Kind: "InfrastructureStatus", } +// RenderTerraformerChart renders the azure-infra chart with the given values. +func RenderTerraformerChart(renderer chartrenderer.Interface, infra *extensionsv1alpha1.Infrastructure, clientAuth *internal.ClientAuth, + config *api.InfrastructureConfig, cluster *controller.Cluster) (*TerraformFiles, error) { + values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) + if err != nil { + return nil, err + } + + release, err := renderer.Render(filepath.Join(azure.InternalChartsPath, "azure-infra"), "azure-infra", infra.Namespace, values) + if err != nil { + return nil, err + } + + return &TerraformFiles{ + Main: release.FileContent("main.tf"), + Variables: release.FileContent("variables.tf"), + TFVars: []byte(release.FileContent("terraform.tfvars")), + }, nil +} + // ComputeTerraformerChartValues computes the values for the Azure Terraformer chart. func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, clientAuth *internal.ClientAuth, config *api.InfrastructureConfig, cluster *controller.Cluster) (map[string]interface{}, error) { @@ -78,7 +98,6 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli createResourceGroup = true createVNet = true createAvailabilitySet = false - createNatGateway = false resourceGroupName = infra.Namespace identityConfig map[string]interface{} @@ -97,7 +116,6 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli "routeTableName": TerraformerOutputKeyRouteTableName, "securityGroupName": TerraformerOutputKeySecurityGroupName, } - natGatewayConfig = map[string]interface{}{} ) primaryAvSetRequired, err := isPrimaryAvailabilitySetRequired(infra, config, cluster) @@ -140,12 +158,7 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli azureConfig["countUpdateDomains"] = count.updateDomains } - if config.Networks.NatGateway != nil && config.Networks.NatGateway.Enabled { - createNatGateway = true - if config.Networks.NatGateway.IdleConnectionTimeoutMinutes != nil { - natGatewayConfig["idleConnectionTimeoutMinutes"] = *config.Networks.NatGateway.IdleConnectionTimeoutMinutes - } - } + natGatewayConfig, createNatGateway := generateNatGatewayValues(config) // Checks if the Gardener managed NatGateway public ip needs to be migrated. // TODO(natipmigration) This can be removed in future versions when the ip migration has been completed. @@ -189,24 +202,32 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli }, nil } -// RenderTerraformerChart renders the azure-infra chart with the given values. -func RenderTerraformerChart(renderer chartrenderer.Interface, infra *extensionsv1alpha1.Infrastructure, clientAuth *internal.ClientAuth, - config *api.InfrastructureConfig, cluster *controller.Cluster) (*TerraformFiles, error) { - values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) - if err != nil { - return nil, err +func generateNatGatewayValues(config *api.InfrastructureConfig) (map[string]interface{}, bool) { + var natGatewayConfig = make(map[string]interface{}) + if config.Networks.NatGateway == nil || !config.Networks.NatGateway.Enabled { + return natGatewayConfig, false } - release, err := renderer.Render(filepath.Join(azure.InternalChartsPath, "azure-infra"), "azure-infra", infra.Namespace, values) - if err != nil { - return nil, err + if config.Networks.NatGateway.IdleConnectionTimeoutMinutes != nil { + natGatewayConfig["idleConnectionTimeoutMinutes"] = *config.Networks.NatGateway.IdleConnectionTimeoutMinutes } - return &TerraformFiles{ - Main: release.FileContent("main.tf"), - Variables: release.FileContent("variables.tf"), - TFVars: []byte(release.FileContent("terraform.tfvars")), - }, nil + if config.Networks.NatGateway.Zone != nil { + natGatewayConfig["zone"] = *config.Networks.NatGateway.Zone + } + + if len(config.Networks.NatGateway.IPAddresses) > 0 { + var ipAddresses = make([]map[string]interface{}, len(config.Networks.NatGateway.IPAddresses)) + for i, ip := range config.Networks.NatGateway.IPAddresses { + ipAddresses[i] = map[string]interface{}{ + "name": ip.Name, + "resourceGroup": ip.ResourceGroup, + } + } + natGatewayConfig["ipAddresses"] = ipAddresses + } + + return natGatewayConfig, true } // TerraformFiles are the files that have been rendered from the infrastructure chart. diff --git a/pkg/internal/infrastructure/terraform_test.go b/pkg/internal/infrastructure/terraform_test.go index e74d69a55..4790a4ef7 100644 --- a/pkg/internal/infrastructure/terraform_test.go +++ b/pkg/internal/infrastructure/terraform_test.go @@ -387,7 +387,7 @@ var _ = Describe("Terraform", func() { } expectedCreateValues["natGateway"] = true values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) - Expect(err).To(Not(HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) Expect(values).To(BeEquivalentTo(expectedValues)) }) @@ -400,7 +400,46 @@ var _ = Describe("Terraform", func() { expectedCreateValues["natGateway"] = true expectedNatGatewayValues["idleConnectionTimeoutMinutes"] = timeout values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) - Expect(err).To(Not(HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(BeEquivalentTo(expectedValues)) + }) + + It("should correctly compute terraform chart values with zonal NatGateway", func() { + config.Networks.NatGateway = &api.NatGatewayConfig{ + Enabled: true, + Zone: pointer.Int32Ptr(1), + } + expectedCreateValues["natGateway"] = true + expectedNatGatewayValues["zone"] = int32(1) + values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) + Expect(err).NotTo(HaveOccurred()) + Expect(values).To(BeEquivalentTo(expectedValues)) + }) + + It("should correctly compute terraform chart values with zonal public ip addresses", func() { + var ( + ipName = "public-ip-1-name" + ipResourceGroup = "public-ip-1-resource-group" + ) + + config.Networks.NatGateway = &api.NatGatewayConfig{ + Enabled: true, + Zone: pointer.Int32Ptr(1), + IPAddresses: []api.PublicIPReference{{ + Name: ipName, + ResourceGroup: ipResourceGroup, + Zone: int32(1), + }}, + } + expectedCreateValues["natGateway"] = true + expectedNatGatewayValues["zone"] = int32(1) + expectedNatGatewayValues["ipAddresses"] = []map[string]interface{}{{ + "name": ipName, + "resourceGroup": ipResourceGroup, + }} + + values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster) + Expect(err).NotTo(HaveOccurred()) Expect(values).To(BeEquivalentTo(expectedValues)) })