Skip to content

Commit

Permalink
NatGateway Integration – Step 2
Browse files Browse the repository at this point in the history
  • Loading branch information
@dkistner
dkistner committed Jan 21, 2021
1 parent 9363d41 commit 012f0da
Show file tree
Hide file tree
Showing 14 changed files with 529 additions and 142 deletions.
39 changes: 39 additions & 0 deletions charts/internal/azure-infra/templates/_helpers.tpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{{- define "resource-group-reference" -}}
{{- if .Values.create.resourceGroup -}}
azurerm_resource_group.rg.name
{{- else -}}
data.azurerm_resource_group.rg.name
{{- end}}
{{- end -}}

{{- define "natgateway-managed-ip" -}}
# Gardener managed public IP to be attached to the NatGateway.
resource "azurerm_public_ip" "natip" {
name = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
location = "{{ required "azure.region is required" .Values.azure.region }}"
resource_group_name = {{ template "resource-group-reference" . }}
allocation_method = "Static"
sku = "Standard"
{{ if .Values.natGateway -}}{{ if hasKey .Values.natGateway "zone" -}}
zones = [{{ .Values.natGateway.zone | quote }}]
{{- end }}{{- end }}
}
resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
nat_gateway_id = azurerm_nat_gateway.nat.id
public_ip_address_id = azurerm_public_ip.natip.id
}
{{- end -}}

{{- define "natgateway-user-provided-public-ips" -}}
# User provided public IPs to be attached to the NatGateway.
{{ range $index, $ip := .Values.natGateway.ipAddresses -}}
data "azurerm_public_ip" "natip-user-provided-{{ $index }}" {
name = "{{ $ip.name }}"
resource_group_name = "{{ $ip.resourceGroup }}"
}
resource "azurerm_nat_gateway_public_ip_association" "natip-user-provided-{{ $index }}-association" {
nat_gateway_id = azurerm_nat_gateway.nat.id
public_ip_address_id = data.azurerm_public_ip.natip-user-provided-{{ $index }}.id
}
{{ end }}
{{- end -}}
81 changes: 28 additions & 53 deletions charts/internal/azure-infra/templates/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ provider "azurerm" {
features {}
}

#===============================================
#= Resource Group
#===============================================
{{ if .Values.create.resourceGroup -}}
resource "azurerm_resource_group" "rg" {
name = "{{ required "resourceGroup.name is required" .Values.resourceGroup.name }}"
Expand All @@ -22,14 +25,11 @@ data "azurerm_resource_group" "rg" {
#= VNet, Subnets, Route Table, Security Groups
#===============================================

# VNet
{{ if .Values.create.vnet -}}
resource "azurerm_virtual_network" "vnet" {
name = "{{ required "resourceGroup.vnet.name is required" .Values.resourceGroup.vnet.name }}"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end}}
resource_group_name = {{ template "resource-group-reference" . }}
location = "{{ required "azure.region is required" .Values.azure.region }}"
address_space = ["{{ required "resourceGroup.vnet.cidr is required" .Values.resourceGroup.vnet.cidr }}"]
}
Expand All @@ -40,6 +40,7 @@ data "azurerm_virtual_network" "vnet" {
}
{{- end }}

# Subnet
resource "azurerm_subnet" "workers" {
name = "{{ required "clusterName is required" .Values.clusterName }}-nodes"
{{ if .Values.create.vnet -}}
Expand All @@ -53,31 +54,23 @@ resource "azurerm_subnet" "workers" {
service_endpoints = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}]
}

# RouteTable
resource "azurerm_route_table" "workers" {
name = "worker_route_table"
location = "{{ required "azure.region is required" .Values.azure.region }}"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end}}
resource_group_name = {{ template "resource-group-reference" . }}
}

resource "azurerm_subnet_route_table_association" "workers-rt-subnet-association" {
subnet_id = azurerm_subnet.workers.id
route_table_id = azurerm_route_table.workers.id
}

# SecurityGroup
resource "azurerm_network_security_group" "workers" {
name = "{{ required "clusterName is required" .Values.clusterName }}-workers"
location = "{{ required "azure.region is required" .Values.azure.region }}"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end}}
resource_group_name = {{ template "resource-group-reference" . }}
}

resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet-association" {
subnet_id = azurerm_subnet.workers.id
network_security_group_id = azurerm_network_security_group.workers.id
Expand All @@ -88,48 +81,38 @@ resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet
#= NAT Gateway
#===============================================

resource "azurerm_public_ip" "natip" {
name = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
location = "{{ required "azure.region is required" .Values.azure.region }}"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end }}
allocation_method = "Static"
sku = "Standard"
}

resource "azurerm_nat_gateway" "nat" {
name = "{{ required "clusterName is required" .Values.clusterName }}-nat-gateway"
location = "{{ required "azure.region is required" .Values.azure.region }}"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end }}
resource_group_name = {{ template "resource-group-reference" . }}
sku_name = "Standard"
{{ if .Values.natGateway -}}
{{ if .Values.natGateway.idleConnectionTimeoutMinutes -}}
{{ if hasKey .Values.natGateway "idleConnectionTimeoutMinutes" -}}
idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }}
{{- end }}

# TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
{{ if hasKey .Values.natGateway "zone" -}}
zones = [{{ .Values.natGateway.zone | quote }}]
{{- end }}
{{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
# TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
public_ip_address_ids = []
{{- end }}
{{- end }}
}

resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
nat_gateway_id = azurerm_nat_gateway.nat.id
public_ip_address_id = azurerm_public_ip.natip.id
}

resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
subnet_id = azurerm_subnet.workers.id
nat_gateway_id = azurerm_nat_gateway.nat.id
}

{{ if .Values.natGateway -}}
{{ if and (hasKey .Values.natGateway "ipAddresses") (hasKey .Values.natGateway "zone") -}}
{{ template "natgateway-user-provided-public-ips" . }}
{{- else -}}
{{ template "natgateway-managed-ip" . }}
{{- end }}
{{- else -}}
{{ template "natgateway-managed-ip" . }}
{{- end }}
{{- end }}

{{ if .Values.identity -}}
Expand All @@ -150,12 +133,8 @@ data "azurerm_user_assigned_identity" "identity" {

resource "azurerm_availability_set" "workers" {
name = "{{ required "clusterName is required" .Values.clusterName }}-avset-workers"
{{ if .Values.create.resourceGroup -}}
resource_group_name = azurerm_resource_group.rg.name
{{- else -}}
resource_group_name = data.azurerm_resource_group.rg.name
{{- end}}
location = "{{ required "azure.region is required" .Values.azure.region }}"
resource_group_name = {{ template "resource-group-reference" . }}
platform_update_domain_count = "{{ required "azure.countUpdateDomains is required" .Values.azure.countUpdateDomains }}"
platform_fault_domain_count = "{{ required "azure.countFaultDomains is required" .Values.azure.countFaultDomains }}"
managed = true
Expand All @@ -167,11 +146,7 @@ resource "azurerm_availability_set" "workers" {
#===============================================

output "{{ .Values.outputKeys.resourceGroupName }}" {
{{- if .Values.create.resourceGroup }}
value = azurerm_resource_group.rg.name
{{- else -}}
value = data.azurerm_resource_group.rg.name
{{- end}}
value = {{ template "resource-group-reference" . }}
}

{{ if .Values.create.vnet -}}
Expand Down
13 changes: 9 additions & 4 deletions charts/internal/azure-infra/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,15 @@ create:
# name: identity-name
# resourceGroup: identity-resource-group

natGateway:
idleConnectionTimeoutMinutes:
# TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
migrateNatGatewayToIPAssociation: false
# natGateway:
# idleConnectionTimeoutMinutes: 4
# zone: 1
# migrateNatGatewayToIPAssociation: false # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
# ipAddresses:
# - name: public-ip-0-name
# resourceGroup: public-ip-0-resource-group
# - name: public-ip-1-name
# resourceGroup: public-ip-1-resource-group

resourceGroup:
name: my-resource-group
Expand Down
31 changes: 21 additions & 10 deletions docs/usage-as-end-user.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ networks:
# natGateway:
# enabled: false
# idleConnectionTimeoutMinutes: 4
# zone: 1
# ipAddresses:
# - name: my-public-ip-name
# resourceGroup: my-public-ip-resource-group
# zone: 1
# serviceEndpoints:
# - Microsoft.Test
zoned: false
Expand All @@ -62,6 +67,13 @@ zoned: false
# acrAccess: true
```

Currently, it's not yet possible to deploy into existing resource groups, but in the future it will.
The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to.

Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
If you don't use zones then an availability set will be created and only basic load balancers will be used.
Zoned clusters use standard load balancers.

The `networks.vnet` section describes whether you want to create the shoot cluster in an already existing VNet or whether to create a new one:

* If `networks.vnet.name` and `networks.vnet.resourceGroup` are given then you have to specify the VNet name and VNet resource group name of the existing VNet that was created by other means (manually, other tooling, ...).
Expand All @@ -75,18 +87,17 @@ You can freely choose this CIDR and it is your responsibility to properly design

In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview).

The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. If NAT Gateway is enabled, the field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers).

Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
If you don't use zones then an availability set will be created and only basic load balancers will be used.
Zoned clusters use standard load balancers.
The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of a Shoot cluster. Here are some key information about the usage of the NatGateway for a Shoot cluster:
- NatGateway usage is optional and can be enabled or disabled via `.networks.natGateway.enabled`.
- If the NatGateway is not used then the egress connections initiated within the Shoot cluster will be nated via the LoadBalancer of the clusters (default Azure behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)).
- NatGateway is only available for zonal clusters `.zoned=true`.
- The NatGateway is currently **not** zone redundantly deployed. That mean the NatGateway of a Shoot cluster will always be in just one zone. Those zone can be optionally selected via `networks.natGateway.zone`.
- **Caution:** Modifing the `networks.natGateway.zone` setting require a recreation of the NatGateway and the managed public ip (automatically used if no own public ip is specified, see below). That mean you will most likely get a different public ip for egress connections.
- It is possible to bring own zonal public ip(s) via `networks.natGateway.ipAddresses`. Those public ip(s) need to be in the same zone as the NatGateway (see `networks.natGateway.zone`) and be of SKU `standard`. For each public the `name`, the `resourceGroup` and the `zone` need to be specified.
- The field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers).

In the `identity` section you can specify an [Azure user-assigned managed identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview#how-does-the-managed-identities-for-azure-resources-work) which should be attached to all cluster worker machines. With `identity.name` you can specify the name of the identity and with `identity.resourceGroup` you can specify the resource group which contains the identity resource on Azure. The identity need to be created by the user upfront (manually, other tooling, ...). Gardener/Azure Extension will only use the referenced one and won't create an identity. Furthermore the identity have to be in the same subscription as the Shoot cluster. Via the `identity.acrAccess` you can configure the worker machines to use the passed identity for pulling from an [Azure Container Registry (ACR)](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro).

Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster.

Currently, it's not yet possible to deploy into existing resource groups, but in the future it will.
The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to.
**Caution:** Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster.

Apart from the VNet and the worker subnet the Azure extension will also create a dedicated resource group, route tables, security groups, and an availability set (if not using zoned clusters).

Expand Down
79 changes: 77 additions & 2 deletions hack/api-reference/api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -909,7 +909,7 @@ bool
<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig</a>)
</p>
<p>
<p>NatGatewayConfig contains configuration for the nat gateway and the attached resources.</p>
<p>NatGatewayConfig contains configuration for the NAT gateway and the attached resources.</p>
</p>
<table>
<thead>
Expand Down Expand Up @@ -938,10 +938,33 @@ int32
</em>
</td>
<td>
<em>(Optional)</em>
<p>IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes.</p>
</td>
</tr>
<tr>
<td>
<code>zone</code></br>
<em>
int32
</em>
</td>
<td>
<p>Zone specifies the zone in which the NAT gateway should be deployed to.</p>
</td>
</tr>
<tr>
<td>
<code>ipAddresses</code></br>
<em>
<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.PublicIPReference">
[]PublicIPReference
</a>
</em>
</td>
<td>
<p>IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig
Expand Down Expand Up @@ -1058,6 +1081,58 @@ VNetStatus
</tr>
</tbody>
</table>
<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.PublicIPReference">PublicIPReference
</h3>
<p>
(<em>Appears on:</em>
<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NatGatewayConfig">NatGatewayConfig</a>)
</p>
<p>
<p>PublicIPReference contains information about a public ip.</p>
</p>
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>name</code></br>
<em>
string
</em>
</td>
<td>
<p>Name is the name of the public ip.</p>
</td>
</tr>
<tr>
<td>
<code>resourceGroup</code></br>
<em>
string
</em>
</td>
<td>
<p>ResourceGroup is the name of the resource group where the public ip is assigned to.</p>
</td>
</tr>
<tr>
<td>
<code>zone</code></br>
<em>
int32
</em>
</td>
<td>
<p>Zone is the zone in which the public ip is deployed to.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.Purpose">Purpose
(<code>string</code> alias)</p></h3>
<p>
Expand Down
Loading

0 comments on commit 012f0da

Please sign in to comment.