WIP: TF Azurerm v2 plus NatGateway pubIP migration

Add migration to Terraform azurerm provider v2.
Add migration for Gardener managed NatGateway public ip.

TODO: This require gardener#161

charts/internal/azure-infra/templates/main.tf

@@ -3,6 +3,8 @@ provider "azurerm" {
   tenant_id       = "{{ required "azure.tenantID is required" .Values.azure.tenantID }}"
   client_id       = var.CLIENT_ID
   client_secret   = var.CLIENT_SECRET
+
+  features {}
 }
 
 {{ if .Values.create.resourceGroup -}}
@@ -47,10 +49,8 @@ resource "azurerm_subnet" "workers" {
   virtual_network_name      = data.azurerm_virtual_network.vnet.name
   resource_group_name       = data.azurerm_virtual_network.vnet.resource_group_name
   {{- end }}
-  address_prefix            = "{{ required "networks.worker is required" .Values.networks.worker }}"
+  address_prefixes          = ["{{ required "networks.worker is required" .Values.networks.worker }}"]
   service_endpoints         = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}]
-  route_table_id            = azurerm_route_table.workers.id
-  network_security_group_id = azurerm_network_security_group.workers.id
 }
 
 resource "azurerm_route_table" "workers" {
@@ -109,14 +109,22 @@ resource "azurerm_nat_gateway" "nat" {
   resource_group_name     = data.azurerm_resource_group.rg.name
   {{- end }}
   sku_name                = "Standard"
-  public_ip_address_ids   = [azurerm_public_ip.natip.id]
-  {{- if .Values.natGateway }}
-  {{- if .Values.natGateway.idleConnectionTimeoutMinutes }}
+  {{ if .Values.natGateway -}}
+  {{ if .Values.natGateway.idleConnectionTimeoutMinutes -}}
   idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }}
   {{- end }}
+  {{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
+  # TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+  public_ip_address_ids   = []
+  {{- end }}
   {{- end }}
 }
 
+resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
+  nat_gateway_id       = azurerm_nat_gateway.nat.id
+  public_ip_address_id = azurerm_public_ip.natip.id
+}
+
 resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
   subnet_id      = azurerm_subnet.workers.id
   nat_gateway_id = azurerm_nat_gateway.nat.id
@@ -216,4 +224,11 @@ output "{{ .Values.outputKeys.identityID }}" {
 output "{{ .Values.outputKeys.identityClientID }}" {
   value = data.azurerm_user_assigned_identity.identity.client_id
 }
-{{- end }}
+{{- end }}
+
+{{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
+# TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+output "{{ .Values.outputKeys.migrateNatGatewayToIPAssociation }}" {
+  value = true
+}
+{{- end }}

charts/internal/azure-infra/values.yaml

@@ -15,6 +15,11 @@ create:
   # name: identity-name
   # resourceGroup: identity-resource-group
 
+natGateway:
+  idleConnectionTimeoutMinutes:
+  # TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+  migrateNatGatewayToIPAssociation: false
+
 resourceGroup:
   name: my-resource-group
   vnet:
@@ -42,7 +47,7 @@ outputKeys:
   securityGroupName: securityGroupName
   # identityID: managedIdentityID
   # identityClientID: managedIdentityClientID
+  # TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+  migrateNatGatewayToIPAssociation: migrateNatGatewayToIPAssociation
 
-natGateway:
-  idleConnectionTimeoutMinutes:
 

hack/api-reference/api.md

@@ -645,6 +645,19 @@ bool
 <p>Zoned indicates whether the cluster uses zones</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>natGatewayPublicIpMigrated</code></br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+TODO(natipmigration) This can be removed in future versions when the ip migration is done.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.MachineImage">MachineImage

pkg/apis/azure/types_infrastructure.go

@@ -70,6 +70,9 @@ type InfrastructureStatus struct {
 	Identity *IdentityStatus
 	// Zoned indicates whether the cluster uses zones
 	Zoned bool
+	// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	NatGatewayPublicIPMigrated bool
 }
 
 // NetworkStatus is the current status of the infrastructure networks.

pkg/apis/azure/v1alpha1/types_infrastructure.go

@@ -78,6 +78,10 @@ type InfrastructureStatus struct {
 	// Zoned indicates whether the cluster uses zones
 	// +optional
 	Zoned bool `json:"zoned,omitempty"`
+	// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	// +optional
+	NatGatewayPublicIPMigrated bool `json:"natGatewayPublicIpMigrated,omitempty"`
 }
 
 // NetworkStatus is the current status of the infrastructure networks.

pkg/controller/infrastructure/actuator.go

@@ -51,7 +51,7 @@ func (a *actuator) updateProviderStatus(
 	infra *extensionsv1alpha1.Infrastructure,
 	config *api.InfrastructureConfig,
 ) error {
-	status, err := infrainternal.ComputeStatus(tf, config)
+	status, err := infrainternal.ComputeStatus(tf, infra, config)
 	if err != nil {
 		return err
 	}

pkg/internal/infrastructure/terraform.go

@@ -61,6 +61,10 @@ const (
 	TerraformerOutputKeyIdentityID = "identityID"
 	// TerraformerOutputKeyIdentityClientID is the key for the identityClientID output
 	TerraformerOutputKeyIdentityClientID = "identityClientID"
+
+	// TerraformerOutputKeyNatGatewayIPMigrated is the key for the migrateNatGatewayToIPAssociation output
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	TerraformerOutputKeyNatGatewayIPMigrated = "migrateNatGatewayToIPAssociation"
 )
 
 // StatusTypeMeta is the TypeMeta of the Azure InfrastructureStatus
@@ -140,6 +144,14 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli
 		}
 	}
 
+	// Checks if the Gardener managed NatGateway public ip needs to be migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	if natGatewayIPMigrationRequired, err := natGatewayPublicIPMigrationRequired(infra, config); err != nil {
+		return nil, err
+	} else if natGatewayIPMigrationRequired {
+		natGatewayConfig["migrateNatGatewayToIPAssociation"] = true
+	}
+
 	if config.Identity != nil && config.Identity.Name != "" && config.Identity.ResourceGroup != "" {
 		identityConfig = map[string]interface{}{
 			"name":          config.Identity.Name,
@@ -227,10 +239,13 @@ type TerraformState struct {
 	IdentityID string
 	// IdentityClientID is the client id of the identity.
 	IdentityClientID string
+	// NatGatewayIPMigrated is the indicator if the nat gateway ip is migrated.
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	NatGatewayIPMigrated string
 }
 
 // ExtractTerraformState extracts the TerraformState from the given Terraformer.
-func ExtractTerraformState(tf terraformer.Terraformer, config *api.InfrastructureConfig) (*TerraformState, error) {
+func ExtractTerraformState(tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (*TerraformState, error) {
 	var outputKeys = []string{
 		TerraformerOutputKeyResourceGroupName,
 		TerraformerOutputKeyRouteTableName,
@@ -252,6 +267,15 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur
 		outputKeys = append(outputKeys, TerraformerOutputKeyIdentityID, TerraformerOutputKeyIdentityClientID)
 	}
 
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	natGatewayIPMigrationRequired, err := natGatewayPublicIPMigrationRequired(infra, config)
+	if err != nil {
+		return nil, err
+	}
+	if natGatewayIPMigrationRequired {
+		outputKeys = append(outputKeys, TerraformerOutputKeyNatGatewayIPMigrated)
+	}
+
 	vars, err := tf.GetStateOutputVariables(outputKeys...)
 	if err != nil {
 		return nil, err
@@ -289,6 +313,11 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur
 		tfState.IdentityClientID = vars[TerraformerOutputKeyIdentityClientID]
 	}
 
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	if natGatewayIPMigrationRequired {
+		tfState.NatGatewayIPMigrated = vars[TerraformerOutputKeyNatGatewayIPMigrated]
+	}
+
 	return &tfState, nil
 }
 
@@ -344,12 +373,17 @@ func StatusFromTerraformState(state *TerraformState) *apiv1alpha1.Infrastructure
 		})
 	}
 
+	// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+	if state.NatGatewayIPMigrated == "true" {
+		tfState.NatGatewayPublicIPMigrated = true
+	}
+
 	return &tfState
 }
 
 // ComputeStatus computes the status based on the Terraformer and the given InfrastructureConfig.
-func ComputeStatus(tf terraformer.Terraformer, config *api.InfrastructureConfig) (*apiv1alpha1.InfrastructureStatus, error) {
-	state, err := ExtractTerraformState(tf, config)
+func ComputeStatus(tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (*apiv1alpha1.InfrastructureStatus, error) {
+	state, err := ExtractTerraformState(tf, infra, config)
 	if err != nil {
 		return nil, err
 	}
@@ -418,3 +452,24 @@ func findDomainCounts(cluster *controller.Cluster, infra *extensionsv1alpha1.Inf
 		updateDomains: *updateDomainCount,
 	}, nil
 }
+
+// natGatewayPublicIPMigrationRequired checks if the Gardener managed NatGateway public ip needs to be migrated.
+// TODO(natipmigration) This can be removed in future versions when the ip migration is done.
+func natGatewayPublicIPMigrationRequired(infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (bool, error) {
+	if config.Networks.NatGateway == nil || !config.Networks.NatGateway.Enabled {
+		return false, nil
+	}
+
+	// Check if the natgateway ip is already migrated.
+	if infra.Status.ProviderStatus != nil {
+		infrastructureStatus, err := helper.InfrastructureStatusFromInfrastructure(infra)
+		if err != nil {
+			return false, err
+		}
+		if infrastructureStatus.NatGatewayPublicIPMigrated {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}

pkg/internal/terraform.go

@@ -17,7 +17,6 @@ package internal
 import (
 	"time"
 
-	"github.com/gardener/gardener-extension-provider-azure/pkg/internal/imagevector"
 	"github.com/gardener/gardener/extensions/pkg/terraformer"
 
 	"github.com/gardener/gardener/pkg/logger"
@@ -47,7 +46,8 @@ func NewTerraformer(
 	namespace,
 	name string,
 ) (terraformer.Terraformer, error) {
-	tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, imagevector.TerraformerImage())
+	tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, "dominickistner/terraformer:az-2-12-0-nr2")
+	//tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, imagevector.TerraformerImage())
 	if err != nil {
 		return nil, err
 	}