From ee7f89db8e36ee1a3fd7959e10106a17a0516694 Mon Sep 17 00:00:00 2001 From: Yury Krasouski Date: Tue, 7 May 2024 13:51:58 +0200 Subject: [PATCH] Feature #4784: SubnetSchema support --- api/v1beta1/awscluster_conversion.go | 1 + api/v1beta1/zz_generated.conversion.go | 1 + api/v1beta2/awscluster_webhook_test.go | 1 + api/v1beta2/network_types.go | 10 + api/v1beta2/types.go | 18 + api/v1beta2/zz_generated.deepcopy.go | 5 + ...ster.x-k8s.io_awsmanagedcontrolplanes.yaml | 24 + ...tructure.cluster.x-k8s.io_awsclusters.yaml | 12 + ....cluster.x-k8s.io_awsclustertemplates.yaml | 12 + .../awsmanagedcontrolplane_webhook_test.go | 1 + pkg/cloud/services/network/subnets.go | 47 +- pkg/cloud/services/network/subnets_test.go | 756 ++++++++++++++++++ 12 files changed, 876 insertions(+), 12 deletions(-) diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go index 65e797ffba..b3fbc7fd17 100644 --- a/api/v1beta1/awscluster_conversion.go +++ b/api/v1beta1/awscluster_conversion.go @@ -104,6 +104,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup = restored.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch dst.Spec.NetworkSpec.VPC.CarrierGatewayID = restored.Spec.NetworkSpec.VPC.CarrierGatewayID + dst.Spec.NetworkSpec.VPC.SubnetSchema = restored.Spec.NetworkSpec.VPC.SubnetSchema if restored.Spec.NetworkSpec.VPC.ElasticIPPool != nil { if dst.Spec.NetworkSpec.VPC.ElasticIPPool == nil { diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 565df1c3aa..153fba532e 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -2316,6 +2316,7 @@ func autoConvert_v1beta2_VPCSpec_To_v1beta1_VPCSpec(in *v1beta2.VPCSpec, out *VP // WARNING: in.EmptyRoutesDefaultVPCSecurityGroup requires manual conversion: does not exist in peer-type // WARNING: in.PrivateDNSHostnameTypeOnLaunch requires manual conversion: does not exist in peer-type // WARNING: in.ElasticIPPool requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetSchema requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index 54512b2d6c..5252054f7c 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -1062,6 +1062,7 @@ func TestAWSClusterDefaultCNIIngressRules(t *testing.T) { defaultVPCSpec := VPCSpec{ AvailabilityZoneUsageLimit: &AZUsageLimit, AvailabilityZoneSelection: &AZSelectionSchemeOrdered, + SubnetSchema: &SubnetSchemaPreferPrivate, } g := NewWithT(t) tests := []struct { diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 94442ce6f4..2e123cbc35 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -461,6 +461,16 @@ type VPCSpec struct { // the API Server. // +optional ElasticIPPool *ElasticIPPool `json:"elasticIpPool,omitempty"` + + // SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + // PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + // PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + // that will be further sub-divided for the private subnets. + // Defaults to PreferPrivate + // +optional + // +kubebuilder:default=PreferPrivate + // +kubebuilder:validation:Enum=PreferPrivate;PreferPublic + SubnetSchema *SubnetSchemaType `json:"subnetSchema,omitempty"` } // String returns a string representation of the VPC. diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index abf92ae4e0..14e4dcb2e4 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta2 import ( + "strings" + "k8s.io/apimachinery/pkg/util/sets" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -439,3 +441,19 @@ type PrivateDNSName struct { // +kubebuilder:validation:Enum:=ip-name;resource-name HostnameType *string `json:"hostnameType,omitempty"` } + +// SubnetSchemaType specifies how given network should be divided on subnets +// in the VPC depending on the number of AZs. +type SubnetSchemaType string + +// Name returns subnet schema type name without prefix. +func (s *SubnetSchemaType) Name() string { + return strings.ToLower(strings.TrimPrefix(string(*s), "Prefer")) +} + +var ( + // SubnetSchemaPreferPrivate allocates more subnets in the VPC to private subnets. + SubnetSchemaPreferPrivate = SubnetSchemaType("PreferPrivate") + // SubnetSchemaPreferPublic allocates more subnets in the VPC to public subnets. + SubnetSchemaPreferPublic = SubnetSchemaType("PreferPublic") +) diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index d3e08bbb0c..8084ec3e51 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -2192,6 +2192,11 @@ func (in *VPCSpec) DeepCopyInto(out *VPCSpec) { *out = new(ElasticIPPool) (*in).DeepCopyInto(*out) } + if in.SubnetSchema != nil { + in, out := &in.SubnetSchema, &out.SubnetSchema + *out = new(SubnetSchemaType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCSpec. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 7ab077b658..a713e76d54 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -759,6 +759,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string @@ -2750,6 +2762,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index cadff4f653..e59c1f8f78 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1699,6 +1699,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index c353bcff30..147c7e5fe9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -1298,6 +1298,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go index bc3cd5d086..f7210bfb67 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go @@ -45,6 +45,7 @@ func TestDefaultingWebhook(t *testing.T) { defaultVPCSpec := infrav1.VPCSpec{ AvailabilityZoneUsageLimit: &AZUsageLimit, AvailabilityZoneSelection: &infrav1.AZSelectionSchemeOrdered, + SubnetSchema: &infrav1.SubnetSchemaPreferPrivate, } defaultIdentityRef := &infrav1.AWSIdentityReference{ Kind: infrav1.ControllerIdentityKind, diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index f125c7486e..62277cd9fb 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -275,26 +275,38 @@ func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { s.scope.Debug("zones selected", "region", s.scope.Region(), "zones", zones) } - // 1 private subnet for each AZ plus 1 other subnet that will be further sub-divided for the public subnets + // 1 private subnet for each AZ plus 1 other subnet that will be further sub-divided for the public subnets or vice versa if + // the subnet schema is set to prefer public subnets. // All subnets will have an ipv4 address for now as well. We aren't supporting ipv6-only yet. numSubnets := len(zones) + 1 var ( - subnetCIDRs []*net.IPNet - publicSubnetCIDRs []*net.IPNet - ipv6SubnetCIDRs []*net.IPNet - publicIPv6SubnetCIDRs []*net.IPNet - privateIPv6SubnetCIDRs []*net.IPNet + subnetCIDRs []*net.IPNet + preferredSubnetCIDRs []*net.IPNet + residualSubnetCIDRs []*net.IPNet + ipv6SubnetCIDRs []*net.IPNet + preferredIPv6SubnetCIDRs []*net.IPNet + residualIPv6SubnetCIDRs []*net.IPNet ) + subnetScheme := infrav1.SubnetSchemaPreferPrivate + if s.scope.VPC().SubnetSchema != nil { + subnetScheme = *s.scope.VPC().SubnetSchema + } + + residualSubnetsName := infrav1.SubnetSchemaPreferPublic.Name() + if subnetScheme == infrav1.SubnetSchemaPreferPublic { + residualSubnetsName = infrav1.SubnetSchemaPreferPrivate.Name() + } + subnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(s.scope.VPC().CidrBlock, numSubnets) if err != nil { return nil, errors.Wrapf(err, "failed splitting VPC CIDR %q into subnets", s.scope.VPC().CidrBlock) } - publicSubnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) + residualSubnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) if err != nil { - return nil, errors.Wrapf(err, "failed splitting CIDR %q into public subnets", subnetCIDRs[0].String()) + return nil, errors.Wrapf(err, "failed splitting CIDR %q into %s subnets", subnetCIDRs[0].String(), residualSubnetsName) } - privateSubnetCIDRs := append(subnetCIDRs[:0], subnetCIDRs[1:]...) + preferredSubnetCIDRs = append(subnetCIDRs[:0], subnetCIDRs[1:]...) if s.scope.VPC().IsIPv6Enabled() { ipv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(s.scope.VPC().IPv6.CidrBlock, numSubnets) @@ -303,12 +315,23 @@ func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { } // We need to take the last, so it doesn't conflict with the rest. The subnetID is increment each time by 1. - publicIPv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String(), len(zones)) + ipv6SubnetCIDRsStr := ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String() + residualIPv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(ipv6SubnetCIDRsStr, len(zones)) if err != nil { - return nil, errors.Wrapf(err, "failed splitting IPv6 CIDR %q into public subnets", ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String()) + return nil, errors.Wrapf(err, "failed splitting IPv6 CIDR %q into %s subnets", ipv6SubnetCIDRsStr, residualSubnetsName) } // TODO: this might need to be the last instead of the first.. - privateIPv6SubnetCIDRs = append(ipv6SubnetCIDRs[:0], ipv6SubnetCIDRs[1:]...) + preferredIPv6SubnetCIDRs = append(ipv6SubnetCIDRs[:0], ipv6SubnetCIDRs[1:]...) + } + + // By default, the preferred subnets are the private subnets and the residual subnets are the public subnets. + privateSubnetCIDRs, publicSubnetCIDRs := preferredSubnetCIDRs, residualSubnetCIDRs + privateIPv6SubnetCIDRs, publicIPv6SubnetCIDRs := preferredIPv6SubnetCIDRs, residualIPv6SubnetCIDRs + + // If the subnet schema is set to prefer public, we need to swap the private and public subnets. + if subnetScheme == infrav1.SubnetSchemaPreferPublic { + privateSubnetCIDRs, publicSubnetCIDRs = residualSubnetCIDRs, preferredSubnetCIDRs + privateIPv6SubnetCIDRs, publicIPv6SubnetCIDRs = residualIPv6SubnetCIDRs, preferredIPv6SubnetCIDRs } subnets := infrav1.Subnets{} diff --git a/pkg/cloud/services/network/subnets_test.go b/pkg/cloud/services/network/subnets_test.go index b196536d6b..447b1713b8 100644 --- a/pkg/cloud/services/network/subnets_test.go +++ b/pkg/cloud/services/network/subnets_test.go @@ -3071,6 +3071,762 @@ func TestReconcileSubnets(t *testing.T) { stubMockCreateTagsWithContext(m, "test-cluster", "subnet-az-1a-private", "us-east-1a", "private", false).AnyTimes() }, }, + { + name: "Managed VPC, no existing subnets exist, one az, prefer public subnet schema, expect one private and one public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + firstSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + secondSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(firstSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(secondSubnet) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil) + }, + }, + { + name: "Managed IPv6 VPC, no existing subnets exist, one az, prefer public subnet schema, expect one private and one public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:db8:1234:1a01::/56", + PoolID: "amazon", + }, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + firstSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + secondSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(firstSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(secondSubnet) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil) + }, + }, + { + name: "Managed IPv6 VPC, no existing subnets exist, two az's, prefer public subnet schema, expect two private and two public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:db8:1234:1a01::/56", + PoolID: "amazon", + }, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1b"), + ZoneType: aws.String("availability-zone"), + }, + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + // Zone1 + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1b"}), + })). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1b"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).MaxTimes(2) + + zone1PublicSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.64.0/18"), + AvailabilityZone: aws.String("us-east-1b"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1b"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.64.0/18"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1b"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + zone1PrivateSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/19"), + AvailabilityZone: aws.String("us-east-1b"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a04::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1b"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/19"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a04::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1b"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone1PublicSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone1PrivateSubnet) + + // zone 2 + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + zone2PublicSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/18"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/18"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone1PrivateSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone2PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + + zone2PrivateSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.32.0/19"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a05::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.32.0/19"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a05::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone2PublicSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone2PrivateSubnet) + }, + }, } for _, tc := range testCases {