NatGateway integration – step 1

Azure NatGateway integration for zoned clusters only and without option to attach user provided public ip(s)/range(s) to the NatGateway.
Furthermore the NatGateway is not yet deployed zone redundant.

charts/internal/azure-infra/templates/main.tf

@@ -16,9 +16,9 @@ data "azurerm_resource_group" "rg" {
 }
 {{- end}}
 
-#=====================================================================
-#= VNet, Subnets, Route Table, Security Groups, Identity
-#=====================================================================
+#===============================================
+#= VNet, Subnets, Route Table, Security Groups
+#===============================================
 
 {{ if .Values.create.vnet -}}
 resource "azurerm_virtual_network" "vnet" {
@@ -73,17 +73,56 @@ resource "azurerm_network_security_group" "workers" {
   {{- end}}
 }
 
+{{ if .Values.create.natGateway -}}
+#===============================================
+#= NAT Gateway
+#===============================================
+
+resource "azurerm_public_ip" "natip" {
+  name                = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
+  location            = "{{ required "azure.region is required" .Values.azure.region }}"
+  {{ if .Values.create.resourceGroup -}}
+  resource_group_name = "${azurerm_resource_group.rg.name}"
+  {{- else -}}
+  resource_group_name = "${data.azurerm_resource_group.rg.name}"
+  {{- end }}
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+resource "azurerm_nat_gateway" "nat" {
+  name                    = "{{ required "clusterName is required" .Values.clusterName }}-nat-gateway"
+  location                = "{{ required "azure.region is required" .Values.azure.region }}"
+  {{ if .Values.create.resourceGroup -}}
+  resource_group_name     = "${azurerm_resource_group.rg.name}"
+  {{- else -}}
+  resource_group_name     = "${data.azurerm_resource_group.rg.name}"
+  {{- end }}
+  sku_name                = "Standard"
+  public_ip_address_ids   = ["${azurerm_public_ip.natip.id}"]
+}
+
+resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
+  subnet_id      = "${azurerm_subnet.workers.id}"
+  nat_gateway_id = "${azurerm_nat_gateway.nat.id}"
+}
+{{- end }}
+
 {{ if .Values.identity -}}
+#===============================================
+#= Identity
+#===============================================
+
 data "azurerm_user_assigned_identity" "identity" {
   name                = "{{ required "identity.name is required" .Values.identity.name }}"
   resource_group_name = "{{ required "identity.resourceGroup is required" .Values.identity.resourceGroup }}"
 }
 {{- end }}
 
 {{ if .Values.create.availabilitySet -}}
-#=====================================================================
+#===============================================
 #= Availability Set
-#=====================================================================
+#===============================================
 
 resource "azurerm_availability_set" "workers" {
   name                         = "{{ required "clusterName is required" .Values.clusterName }}-avset-workers"
@@ -99,9 +138,9 @@ resource "azurerm_availability_set" "workers" {
 }
 {{- end}}
 
-//=====================================================================
+#===============================================
 //= Output variables
-//=====================================================================
+#===============================================
 
 output "{{ .Values.outputKeys.resourceGroupName }}" {
 {{ if .Values.create.resourceGroup -}}

charts/internal/azure-infra/values.yaml

@@ -9,6 +9,7 @@ create:
   resourceGroup: true
   vnet: true
   availabilitySet: false
+  natGateway: false
 
 # identity:
   # name: identity-name

docs/usage-as-end-user.md

@@ -40,6 +40,8 @@ networks:
     # resouceGroup: my-vnet-resource-group
     cidr: 10.250.0.0/16
   workers: 10.250.0.0/19
+  # natGateway:
+  #   enabled: false
   # serviceEndpoints:
   # - Microsoft.Test
 zoned: false
@@ -64,6 +66,8 @@ You can freely choose this CIDR and it is your responsibility to properly design
 
 In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview).
 
+The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. Furthermore, the Azure NatGateway is not yet generally available (GA) from Azure side, hence, you need to register your subscription to participate in the preview for NatGateway.
+
 Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
 If you don't use zones then an availability set will be created and only basic load balancers will be used.
 Zoned clusters use standard load balancers.

example/30-infrastructure.yaml

@@ -68,6 +68,8 @@ spec:
       # resourceGroup: my-vnet-group
         cidr: 10.250.0.0/16
       workers: 10.250.0.0/19
+    # natGateway:
+    #   enabled: false
     # serviceEndpoints:
     # - entry1
     zoned: false

hack/api-reference/api.md

@@ -746,6 +746,36 @@ string
 </tr>
 </tbody>
 </table>
+<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.NatGatewayConfig">NatGatewayConfig
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig</a>)
+</p>
+<p>
+<p>NatGatewayConfig contains configuration for the nat gateway and the attached resources.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>enabled</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<p>Enabled is an indicator if NAT gateway should be deployed.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig
 </h3>
 <p>
@@ -789,6 +819,20 @@ string
 </tr>
 <tr>
 <td>
+<code>natGateway</code></br>
+<em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NatGatewayConfig">
+NatGatewayConfig
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>NatGateway contains the configuration for the NatGateway.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>serviceEndpoints</code></br>
 <em>
 []string

pkg/apis/azure/types_infrastructure.go

@@ -45,6 +45,8 @@ type NetworkConfig struct {
 	VNet VNet
 	// Workers is the worker subnet range to create (used for the VMs).
 	Workers string
+	// NatGateway contains the configuration for the NatGateway.
+	NatGateway *NatGatewayConfig
 	// ServiceEndpoints is a list of Azure ServiceEndpoints which should be associated with the worker subnet.
 	ServiceEndpoints []string
 }
@@ -140,6 +142,12 @@ type VNetStatus struct {
 	ResourceGroup *string
 }
 
+// NatGatewayConfig contains configuration for the nat gateway and the attached resources.
+type NatGatewayConfig struct {
+	// Enabled is an indicator if NAT gateway should be deployed.
+	Enabled bool
+}
+
 // IdentityConfig contains configuration for the managed identity.
 type IdentityConfig struct {
 	// Name is the name of the identity.

pkg/apis/azure/v1alpha1/types_infrastructure.go

@@ -49,6 +49,9 @@ type NetworkConfig struct {
 	VNet VNet `json:"vnet"`
 	// Workers is the worker subnet range to create (used for the VMs).
 	Workers string `json:"workers"`
+	// NatGateway contains the configuration for the NatGateway.
+	// +optional
+	NatGateway *NatGatewayConfig `json:"natGateway,omitempty"`
 	// ServiceEndpoints is a list of Azure ServiceEndpoints which should be associated with the worker subnet.
 	// +optional
 	ServiceEndpoints []string `json:"serviceEndpoints,omitempty"`
@@ -152,6 +155,12 @@ type VNetStatus struct {
 	ResourceGroup *string `json:"resourceGroup,omitempty"`
 }
 
+// NatGatewayConfig contains configuration for the nat gateway and the attached resources.
+type NatGatewayConfig struct {
+	// Enabled is an indicator if NAT gateway should be deployed.
+	Enabled bool `json:"enabled"`
+}
+
 // IdentityConfig contains configuration for the managed identity.
 type IdentityConfig struct {
 	// Name is the name of the identity.

pkg/apis/azure/validation/infrastructure.go

@@ -83,6 +83,16 @@ func ValidateInfrastructureConfig(infra *apisazure.InfrastructureConfig, nodesCI
 		}
 	}
 
+	// TODO(dkistner) Remove once we proceed with multiple AvailabilitySet support.
+	// Currently we will not offer Nat Gateway for non zoned/AvailabilitySet based
+	// clusters as the NatGateway is not compatible with Basic LoadBalancer and
+	// we would need Standard LoadBalancers also in combination with AvailabilitySets.
+	// For the multiple AvailabilitySet approach we would always need
+	// a Standard LoadBalancer and a NatGateway.
+	if !infra.Zoned && infra.Networks.NatGateway != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("networks", "natGateway"), infra.Networks.NatGateway, "NatGateway is currently only supported for zoned cluster"))
+	}
+
 	if infra.Identity != nil && (infra.Identity.Name == "" || infra.Identity.ResourceGroup == "") {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("identity"), infra.Identity, "specifying an identity requires the name of the identity and the resource group which hosts the identity"))
 	}

pkg/apis/azure/validation/infrastructure_test.go

@@ -270,6 +270,26 @@ var _ = Describe("InfrastructureConfig validation", func() {
 				}))
 			})
 		})
+
+		Context("NatGateway", func() {
+			It("should return no errors using a NatGateway for a zoned cluster", func() {
+				infrastructureConfig.Zoned = true
+				infrastructureConfig.Networks.NatGateway = &apisazure.NatGatewayConfig{Enabled: true}
+				Expect(ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath)).To(BeEmpty())
+			})
+
+			It("should return an error using a NatGateway for a non zoned cluster", func() {
+				infrastructureConfig.Zoned = false
+				infrastructureConfig.Networks.NatGateway = &apisazure.NatGatewayConfig{}
+				errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath)
+				Expect(errorList).To(HaveLen(1))
+				Expect(errorList).To(ConsistOfFields(Fields{
+					"Type":   Equal(field.ErrorTypeInvalid),
+					"Field":  Equal("networks.natGateway"),
+					"Detail": Equal("NatGateway is currently only supported for zoned cluster"),
+				}))
+			})
+		})
 	})
 
 	Describe("#ValidateInfrastructureConfigUpdate", func() {