From c6e3617f41812b13f4f0835f3114c97803c55cd0 Mon Sep 17 00:00:00 2001 From: dkistner Date: Tue, 8 Sep 2020 10:30:25 +0200 Subject: [PATCH] WIP: TF Azurerm v2 plus NatGateway pubIP migration Add migration to Terraform azurerm provider v2. Add migration for Gardener managed NatGateway public ip. TODO: This require https://github.com/gardener/gardener-extension-provider-azure/pull/161 --- charts/internal/azure-infra/templates/main.tf | 29 ++++++--- charts/internal/azure-infra/values.yaml | 9 ++- hack/api-reference/api.md | 13 ++++ pkg/apis/azure/types_infrastructure.go | 3 + .../azure/v1alpha1/types_infrastructure.go | 4 ++ .../azure/v1alpha1/zz_generated.conversion.go | 2 + pkg/controller/infrastructure/actuator.go | 2 +- pkg/internal/infrastructure/terraform.go | 61 ++++++++++++++++++- pkg/internal/terraform.go | 4 +- 9 files changed, 112 insertions(+), 15 deletions(-) diff --git a/charts/internal/azure-infra/templates/main.tf b/charts/internal/azure-infra/templates/main.tf index feff588dc..1c1a78566 100644 --- a/charts/internal/azure-infra/templates/main.tf +++ b/charts/internal/azure-infra/templates/main.tf @@ -3,6 +3,8 @@ provider "azurerm" { tenant_id = "{{ required "azure.tenantID is required" .Values.azure.tenantID }}" client_id = var.CLIENT_ID client_secret = var.CLIENT_SECRET + + features {} } {{ if .Values.create.resourceGroup -}} @@ -47,10 +49,8 @@ resource "azurerm_subnet" "workers" { virtual_network_name = data.azurerm_virtual_network.vnet.name resource_group_name = data.azurerm_virtual_network.vnet.resource_group_name {{- end }} - address_prefix = "{{ required "networks.worker is required" .Values.networks.worker }}" + address_prefixes = ["{{ required "networks.worker is required" .Values.networks.worker }}"] service_endpoints = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}] - route_table_id = azurerm_route_table.workers.id - network_security_group_id = azurerm_network_security_group.workers.id } resource "azurerm_route_table" "workers" { @@ -109,14 +109,22 @@ resource "azurerm_nat_gateway" "nat" { resource_group_name = data.azurerm_resource_group.rg.name {{- end }} sku_name = "Standard" - public_ip_address_ids = [azurerm_public_ip.natip.id] - {{- if .Values.natGateway }} - {{- if .Values.natGateway.idleConnectionTimeoutMinutes }} + {{ if .Values.natGateway -}} + {{ if .Values.natGateway.idleConnectionTimeoutMinutes -}} idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }} {{- end }} + {{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}} + # TODO(natipmigration) This can be removed in future versions when the ip migration is done. + public_ip_address_ids = [] + {{- end }} {{- end }} } +resource "azurerm_nat_gateway_public_ip_association" "natip-association" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.natip.id +} + resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" { subnet_id = azurerm_subnet.workers.id nat_gateway_id = azurerm_nat_gateway.nat.id @@ -216,4 +224,11 @@ output "{{ .Values.outputKeys.identityID }}" { output "{{ .Values.outputKeys.identityClientID }}" { value = data.azurerm_user_assigned_identity.identity.client_id } -{{- end }} \ No newline at end of file +{{- end }} + +{{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}} +# TODO(natipmigration) This can be removed in future versions when the ip migration is done. +output "{{ .Values.outputKeys.migrateNatGatewayToIPAssociation }}" { + value = true +} +{{- end }} diff --git a/charts/internal/azure-infra/values.yaml b/charts/internal/azure-infra/values.yaml index add75db10..0d00953d4 100644 --- a/charts/internal/azure-infra/values.yaml +++ b/charts/internal/azure-infra/values.yaml @@ -15,6 +15,11 @@ create: # name: identity-name # resourceGroup: identity-resource-group +natGateway: + idleConnectionTimeoutMinutes: + # TODO(natipmigration) This can be removed in future versions when the ip migration is done. + migrateNatGatewayToIPAssociation: false + resourceGroup: name: my-resource-group vnet: @@ -42,7 +47,7 @@ outputKeys: securityGroupName: securityGroupName # identityID: managedIdentityID # identityClientID: managedIdentityClientID + # TODO(natipmigration) This can be removed in future versions when the ip migration is done. + migrateNatGatewayToIPAssociation: migrateNatGatewayToIPAssociation -natGateway: - idleConnectionTimeoutMinutes: diff --git a/hack/api-reference/api.md b/hack/api-reference/api.md index a3af773fc..47e9377be 100644 --- a/hack/api-reference/api.md +++ b/hack/api-reference/api.md @@ -645,6 +645,19 @@ bool

Zoned indicates whether the cluster uses zones

+ + +natGatewayPublicIpMigrated
+ +bool + + + +(Optional) +

NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated. +TODO(natipmigration) This can be removed in future versions when the ip migration is done.

+ +

MachineImage diff --git a/pkg/apis/azure/types_infrastructure.go b/pkg/apis/azure/types_infrastructure.go index 2b409380a..23596f54b 100644 --- a/pkg/apis/azure/types_infrastructure.go +++ b/pkg/apis/azure/types_infrastructure.go @@ -70,6 +70,9 @@ type InfrastructureStatus struct { Identity *IdentityStatus // Zoned indicates whether the cluster uses zones Zoned bool + // NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated. + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + NatGatewayPublicIPMigrated bool } // NetworkStatus is the current status of the infrastructure networks. diff --git a/pkg/apis/azure/v1alpha1/types_infrastructure.go b/pkg/apis/azure/v1alpha1/types_infrastructure.go index 2bc923485..b60470e6e 100644 --- a/pkg/apis/azure/v1alpha1/types_infrastructure.go +++ b/pkg/apis/azure/v1alpha1/types_infrastructure.go @@ -78,6 +78,10 @@ type InfrastructureStatus struct { // Zoned indicates whether the cluster uses zones // +optional Zoned bool `json:"zoned,omitempty"` + // NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated. + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + // +optional + NatGatewayPublicIPMigrated bool `json:"natGatewayPublicIpMigrated,omitempty"` } // NetworkStatus is the current status of the infrastructure networks. diff --git a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go index 28f14f493..af5de4950 100644 --- a/pkg/apis/azure/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/azure/v1alpha1/zz_generated.conversion.go @@ -474,6 +474,7 @@ func autoConvert_v1alpha1_InfrastructureStatus_To_azure_InfrastructureStatus(in out.SecurityGroups = *(*[]azure.SecurityGroup)(unsafe.Pointer(&in.SecurityGroups)) out.Identity = (*azure.IdentityStatus)(unsafe.Pointer(in.Identity)) out.Zoned = in.Zoned + out.NatGatewayPublicIPMigrated = in.NatGatewayPublicIPMigrated return nil } @@ -494,6 +495,7 @@ func autoConvert_azure_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in out.SecurityGroups = *(*[]SecurityGroup)(unsafe.Pointer(&in.SecurityGroups)) out.Identity = (*IdentityStatus)(unsafe.Pointer(in.Identity)) out.Zoned = in.Zoned + out.NatGatewayPublicIPMigrated = in.NatGatewayPublicIPMigrated return nil } diff --git a/pkg/controller/infrastructure/actuator.go b/pkg/controller/infrastructure/actuator.go index 7ec7685c3..57175604e 100644 --- a/pkg/controller/infrastructure/actuator.go +++ b/pkg/controller/infrastructure/actuator.go @@ -51,7 +51,7 @@ func (a *actuator) updateProviderStatus( infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig, ) error { - status, err := infrainternal.ComputeStatus(tf, config) + status, err := infrainternal.ComputeStatus(tf, infra, config) if err != nil { return err } diff --git a/pkg/internal/infrastructure/terraform.go b/pkg/internal/infrastructure/terraform.go index 0e967a1e4..fe7baceb3 100644 --- a/pkg/internal/infrastructure/terraform.go +++ b/pkg/internal/infrastructure/terraform.go @@ -61,6 +61,10 @@ const ( TerraformerOutputKeyIdentityID = "identityID" // TerraformerOutputKeyIdentityClientID is the key for the identityClientID output TerraformerOutputKeyIdentityClientID = "identityClientID" + + // TerraformerOutputKeyNatGatewayIPMigrated is the key for the migrateNatGatewayToIPAssociation output + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + TerraformerOutputKeyNatGatewayIPMigrated = "migrateNatGatewayToIPAssociation" ) // StatusTypeMeta is the TypeMeta of the Azure InfrastructureStatus @@ -140,6 +144,14 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli } } + // Checks if the Gardener managed NatGateway public ip needs to be migrated. + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + if natGatewayIPMigrationRequired, err := natGatewayPublicIPMigrationRequired(infra, config); err != nil { + return nil, err + } else if natGatewayIPMigrationRequired { + natGatewayConfig["migrateNatGatewayToIPAssociation"] = true + } + if config.Identity != nil && config.Identity.Name != "" && config.Identity.ResourceGroup != "" { identityConfig = map[string]interface{}{ "name": config.Identity.Name, @@ -227,10 +239,13 @@ type TerraformState struct { IdentityID string // IdentityClientID is the client id of the identity. IdentityClientID string + // NatGatewayIPMigrated is the indicator if the nat gateway ip is migrated. + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + NatGatewayIPMigrated string } // ExtractTerraformState extracts the TerraformState from the given Terraformer. -func ExtractTerraformState(tf terraformer.Terraformer, config *api.InfrastructureConfig) (*TerraformState, error) { +func ExtractTerraformState(tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (*TerraformState, error) { var outputKeys = []string{ TerraformerOutputKeyResourceGroupName, TerraformerOutputKeyRouteTableName, @@ -252,6 +267,15 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur outputKeys = append(outputKeys, TerraformerOutputKeyIdentityID, TerraformerOutputKeyIdentityClientID) } + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + natGatewayIPMigrationRequired, err := natGatewayPublicIPMigrationRequired(infra, config) + if err != nil { + return nil, err + } + if natGatewayIPMigrationRequired { + outputKeys = append(outputKeys, TerraformerOutputKeyNatGatewayIPMigrated) + } + vars, err := tf.GetStateOutputVariables(outputKeys...) if err != nil { return nil, err @@ -289,6 +313,11 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur tfState.IdentityClientID = vars[TerraformerOutputKeyIdentityClientID] } + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + if natGatewayIPMigrationRequired { + tfState.NatGatewayIPMigrated = vars[TerraformerOutputKeyNatGatewayIPMigrated] + } + return &tfState, nil } @@ -344,12 +373,17 @@ func StatusFromTerraformState(state *TerraformState) *apiv1alpha1.Infrastructure }) } + // TODO(natipmigration) This can be removed in future versions when the ip migration is done. + if state.NatGatewayIPMigrated == "true" { + tfState.NatGatewayPublicIPMigrated = true + } + return &tfState } // ComputeStatus computes the status based on the Terraformer and the given InfrastructureConfig. -func ComputeStatus(tf terraformer.Terraformer, config *api.InfrastructureConfig) (*apiv1alpha1.InfrastructureStatus, error) { - state, err := ExtractTerraformState(tf, config) +func ComputeStatus(tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (*apiv1alpha1.InfrastructureStatus, error) { + state, err := ExtractTerraformState(tf, infra, config) if err != nil { return nil, err } @@ -418,3 +452,24 @@ func findDomainCounts(cluster *controller.Cluster, infra *extensionsv1alpha1.Inf updateDomains: *updateDomainCount, }, nil } + +// natGatewayPublicIPMigrationRequired checks if the Gardener managed NatGateway public ip needs to be migrated. +// TODO(natipmigration) This can be removed in future versions when the ip migration is done. +func natGatewayPublicIPMigrationRequired(infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (bool, error) { + if config.Networks.NatGateway == nil || !config.Networks.NatGateway.Enabled { + return false, nil + } + + // Check if the natgateway ip is already migrated. + if infra.Status.ProviderStatus != nil { + infrastructureStatus, err := helper.InfrastructureStatusFromInfrastructure(infra) + if err != nil { + return false, err + } + if infrastructureStatus.NatGatewayPublicIPMigrated { + return false, nil + } + } + + return true, nil +} diff --git a/pkg/internal/terraform.go b/pkg/internal/terraform.go index eed5435ee..edff9b2d0 100644 --- a/pkg/internal/terraform.go +++ b/pkg/internal/terraform.go @@ -17,7 +17,6 @@ package internal import ( "time" - "github.com/gardener/gardener-extension-provider-azure/pkg/internal/imagevector" "github.com/gardener/gardener/extensions/pkg/terraformer" "github.com/gardener/gardener/pkg/logger" @@ -47,7 +46,8 @@ func NewTerraformer( namespace, name string, ) (terraformer.Terraformer, error) { - tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, imagevector.TerraformerImage()) + tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, "dominickistner/terraformer:az-2-12-0-nr2") + //tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, imagevector.TerraformerImage()) if err != nil { return nil, err }