diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go index 382a4cd4d3..4306c1089e 100644 --- a/api/v1beta1/awscluster_conversion.go +++ b/api/v1beta1/awscluster_conversion.go @@ -104,6 +104,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup = restored.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch dst.Spec.NetworkSpec.VPC.CarrierGatewayID = restored.Spec.NetworkSpec.VPC.CarrierGatewayID + dst.Spec.NetworkSpec.VPC.SubnetSchema = restored.Spec.NetworkSpec.VPC.SubnetSchema // Restore SubnetSpec.ResourceID, SubnetSpec.ParentZoneName, and SubnetSpec.ZoneType fields, if any. for _, subnet := range restored.Spec.NetworkSpec.Subnets { diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index fe6510380b..8c667c9263 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -291,3 +291,14 @@ const ( // AmazonLinuxGPU is the AmazonLinux GPU AMI type. AmazonLinuxGPU EKSAMILookupType = "AmazonLinuxGPU" ) + +// SubnetSchemaType specifies how given network should be divided on subnets +// in the VPC depending on the number of AZs. +type SubnetSchemaType string + +var ( + // SubnetSchemaPreferPrivate allocates more subnets in the VPC to private subnets. + SubnetSchemaPreferPrivate = SubnetSchemaType("PreferPrivate") + // SubnetSchemaPreferPublic allocates more subnets in the VPC to public subnets. + SubnetSchemaPreferPublic = SubnetSchemaType("PreferPublic") +) diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 10842bb9ae..146b9c6588 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -2313,6 +2313,7 @@ func autoConvert_v1beta2_VPCSpec_To_v1beta1_VPCSpec(in *v1beta2.VPCSpec, out *VP out.AvailabilityZoneSelection = (*AZSelectionScheme)(unsafe.Pointer(in.AvailabilityZoneSelection)) // WARNING: in.EmptyRoutesDefaultVPCSecurityGroup requires manual conversion: does not exist in peer-type // WARNING: in.PrivateDNSHostnameTypeOnLaunch requires manual conversion: does not exist in peer-type + // WARNING: in.SubnetSchema requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta2/awscluster_webhook_test.go b/api/v1beta2/awscluster_webhook_test.go index 32021c29a1..4b75550d02 100644 --- a/api/v1beta2/awscluster_webhook_test.go +++ b/api/v1beta2/awscluster_webhook_test.go @@ -993,6 +993,7 @@ func TestAWSClusterDefaultCNIIngressRules(t *testing.T) { defaultVPCSpec := VPCSpec{ AvailabilityZoneUsageLimit: &AZUsageLimit, AvailabilityZoneSelection: &AZSelectionSchemeOrdered, + SubnetSchema: &SubnetSchemaPreferPrivate, } g := NewWithT(t) tests := []struct { diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index cd3042b717..e0b32f57df 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -455,6 +455,15 @@ type VPCSpec struct { // +optional // +kubebuilder:validation:Enum:=ip-name;resource-name PrivateDNSHostnameTypeOnLaunch *string `json:"privateDnsHostnameTypeOnLaunch,omitempty"` + + // SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + // PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + // PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + // that will be further sub-divided for the private subnets. + // Defaults to PreferPrivate + // +kubebuilder:default=PreferPrivate + // +kubebuilder:validation:Enum=PreferPrivate;PreferPublic + SubnetSchema *SubnetSchemaType `json:"subnetSchema,omitempty"` } // String returns a string representation of the VPC. diff --git a/api/v1beta2/types.go b/api/v1beta2/types.go index abf92ae4e0..a6485a329c 100644 --- a/api/v1beta2/types.go +++ b/api/v1beta2/types.go @@ -17,6 +17,8 @@ limitations under the License. package v1beta2 import ( + "strings" + "k8s.io/apimachinery/pkg/util/sets" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -439,3 +441,18 @@ type PrivateDNSName struct { // +kubebuilder:validation:Enum:=ip-name;resource-name HostnameType *string `json:"hostnameType,omitempty"` } + +// SubnetSchemaType specifies how given network should be divided on subnets +// in the VPC depending on the number of AZs. +type SubnetSchemaType string + +func (s *SubnetSchemaType) Name() string { + return strings.ToLower(strings.TrimPrefix(string(*s), "Prefer")) +} + +var ( + // SubnetSchemaPreferPrivate allocates more subnets in the VPC to private subnets. + SubnetSchemaPreferPrivate = SubnetSchemaType("PreferPrivate") + // SubnetSchemaPreferPublic allocates more subnets in the VPC to public subnets. + SubnetSchemaPreferPublic = SubnetSchemaType("PreferPublic") +) diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index 81b8a8d314..15e8adc6ce 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -2157,6 +2157,11 @@ func (in *VPCSpec) DeepCopyInto(out *VPCSpec) { *out = new(string) **out = **in } + if in.SubnetSchema != nil { + in, out := &in.SubnetSchema, &out.SubnetSchema + *out = new(SubnetSchemaType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCSpec. diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index c9ffb5ecd8..90c9257261 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -722,6 +722,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string @@ -2672,6 +2684,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index f2d4b882b5..e6046133e9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1658,6 +1658,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index ccc966dbb2..d638eae387 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -1257,6 +1257,18 @@ spec: - ip-name - resource-name type: string + subnetSchema: + default: PreferPrivate + description: |- + SubnetSchema specifies how CidrBlock should be divided on subnets in the VPC depending on the number of AZs. + PreferPrivate - one private subnet for each AZ plus one other subnet that will be further sub-divided for the public subnets. + PreferPublic - have the reverse logic of PreferPrivate, one public subnet for each AZ plus one other subnet + that will be further sub-divided for the private subnets. + Defaults to PreferPrivate + enum: + - PreferPrivate + - PreferPublic + type: string tags: additionalProperties: type: string diff --git a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go index bc3cd5d086..f7210bfb67 100644 --- a/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go +++ b/controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go @@ -45,6 +45,7 @@ func TestDefaultingWebhook(t *testing.T) { defaultVPCSpec := infrav1.VPCSpec{ AvailabilityZoneUsageLimit: &AZUsageLimit, AvailabilityZoneSelection: &infrav1.AZSelectionSchemeOrdered, + SubnetSchema: &infrav1.SubnetSchemaPreferPrivate, } defaultIdentityRef := &infrav1.AWSIdentityReference{ Kind: infrav1.ControllerIdentityKind, diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index f6406bd833..6fcf912eb8 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -274,26 +274,38 @@ func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { s.scope.Debug("zones selected", "region", s.scope.Region(), "zones", zones) } - // 1 private subnet for each AZ plus 1 other subnet that will be further sub-divided for the public subnets + // 1 private subnet for each AZ plus 1 other subnet that will be further sub-divided for the public subnets or vice versa if + // the subnet schema is set to prefer public subnets. // All subnets will have an ipv4 address for now as well. We aren't supporting ipv6-only yet. numSubnets := len(zones) + 1 var ( - subnetCIDRs []*net.IPNet - publicSubnetCIDRs []*net.IPNet - ipv6SubnetCIDRs []*net.IPNet - publicIPv6SubnetCIDRs []*net.IPNet - privateIPv6SubnetCIDRs []*net.IPNet + subnetCIDRs []*net.IPNet + preferredSubnetCIDRs []*net.IPNet + residualSubnetCIDRs []*net.IPNet + ipv6SubnetCIDRs []*net.IPNet + preferredIPv6SubnetCIDRs []*net.IPNet + residualIPv6SubnetCIDRs []*net.IPNet ) + subnetScheme := infrav1.SubnetSchemaPreferPrivate + if s.scope.VPC().SubnetSchema != nil { + subnetScheme = *s.scope.VPC().SubnetSchema + } + + residualSubnetsName := infrav1.SubnetSchemaPreferPublic.Name() + if subnetScheme == infrav1.SubnetSchemaPreferPublic { + residualSubnetsName = infrav1.SubnetSchemaPreferPrivate.Name() + } + subnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(s.scope.VPC().CidrBlock, numSubnets) if err != nil { return nil, errors.Wrapf(err, "failed splitting VPC CIDR %q into subnets", s.scope.VPC().CidrBlock) } - publicSubnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) + residualSubnetCIDRs, err = cidr.SplitIntoSubnetsIPv4(subnetCIDRs[0].String(), len(zones)) if err != nil { - return nil, errors.Wrapf(err, "failed splitting CIDR %q into public subnets", subnetCIDRs[0].String()) + return nil, errors.Wrapf(err, "failed splitting CIDR %q into %s subnets", subnetCIDRs[0].String(), residualSubnetsName) } - privateSubnetCIDRs := append(subnetCIDRs[:0], subnetCIDRs[1:]...) + preferredSubnetCIDRs = append(subnetCIDRs[:0], subnetCIDRs[1:]...) if s.scope.VPC().IsIPv6Enabled() { ipv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(s.scope.VPC().IPv6.CidrBlock, numSubnets) @@ -302,12 +314,23 @@ func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) { } // We need to take the last, so it doesn't conflict with the rest. The subnetID is increment each time by 1. - publicIPv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String(), len(zones)) + ipv6SubnetCIDRsStr := ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String() + residualIPv6SubnetCIDRs, err = cidr.SplitIntoSubnetsIPv6(ipv6SubnetCIDRsStr, len(zones)) if err != nil { - return nil, errors.Wrapf(err, "failed splitting IPv6 CIDR %q into public subnets", ipv6SubnetCIDRs[len(ipv6SubnetCIDRs)-1].String()) + return nil, errors.Wrapf(err, "failed splitting IPv6 CIDR %q into %s subnets", ipv6SubnetCIDRsStr, residualSubnetsName) } // TODO: this might need to be the last instead of the first.. - privateIPv6SubnetCIDRs = append(ipv6SubnetCIDRs[:0], ipv6SubnetCIDRs[1:]...) + preferredIPv6SubnetCIDRs = append(ipv6SubnetCIDRs[:0], ipv6SubnetCIDRs[1:]...) + } + + // By default, the preferred subnets are the private subnets and the residual subnets are the public subnets. + privateSubnetCIDRs, publicSubnetCIDRs := preferredSubnetCIDRs, residualSubnetCIDRs + privateIPv6SubnetCIDRs, publicIPv6SubnetCIDRs := preferredIPv6SubnetCIDRs, residualIPv6SubnetCIDRs + + // If the subnet schema is set to prefer public, we need to swap the private and public subnets. + if subnetScheme == infrav1.SubnetSchemaPreferPublic { + privateSubnetCIDRs, publicSubnetCIDRs = residualSubnetCIDRs, preferredSubnetCIDRs + privateIPv6SubnetCIDRs, publicIPv6SubnetCIDRs = residualIPv6SubnetCIDRs, preferredIPv6SubnetCIDRs } subnets := infrav1.Subnets{} diff --git a/pkg/cloud/services/network/subnets_test.go b/pkg/cloud/services/network/subnets_test.go index 6daa99c9ca..48577be47c 100644 --- a/pkg/cloud/services/network/subnets_test.go +++ b/pkg/cloud/services/network/subnets_test.go @@ -3071,6 +3071,762 @@ func TestReconcileSubnets(t *testing.T) { stubMockCreateTagsWithContext(m, "test-cluster", "subnet-az-1a-private", "us-east-1a", "private", false).AnyTimes() }, }, + { + name: "Managed VPC, no existing subnets exist, one az, prefer public subnet schema, expect one private and one public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + firstSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + secondSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(firstSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(secondSubnet) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil) + }, + }, + { + name: "Managed IPv6 VPC, no existing subnets exist, one az, prefer public subnet schema, expect one private and one public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:db8:1234:1a01::/56", + PoolID: "amazon", + }, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + firstSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(firstSubnet) + + secondSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/17"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/17"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(firstSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(secondSubnet) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil) + }, + }, + { + name: "Managed IPv6 VPC, no existing subnets exist, two az's, prefer public subnet schema, expect two private and two public from default", + input: NewClusterScope().WithNetwork(&infrav1.NetworkSpec{ + VPC: infrav1.VPCSpec{ + ID: subnetsVPCID, + Tags: infrav1.Tags{ + infrav1.ClusterTagKey("test-cluster"): "owned", + }, + CidrBlock: defaultVPCCidr, + IPv6: &infrav1.IPv6{ + CidrBlock: "2001:db8:1234:1a01::/56", + PoolID: "amazon", + }, + SubnetSchema: &infrav1.SubnetSchemaPreferPublic, + }, + Subnets: []infrav1.SubnetSpec{}, + }), + expect: func(m *mocks.MockEC2APIMockRecorder) { + describeCall := m.DescribeSubnetsWithContext(context.TODO(), gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + }, + })). + Return(&ec2.DescribeSubnetsOutput{}, nil) + + m.DescribeRouteTablesWithContext(context.TODO(), gomock.AssignableToTypeOf(&ec2.DescribeRouteTablesInput{})). + Return(&ec2.DescribeRouteTablesOutput{}, nil) + + m.DescribeNatGatewaysPagesWithContext(context.TODO(), + gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String(subnetsVPCID)}, + }, + { + Name: aws.String("state"), + Values: []*string{aws.String("pending"), aws.String("available")}, + }, + }, + }), + gomock.Any()).Return(nil) + + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1b"), + ZoneType: aws.String("availability-zone"), + }, + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + // Zone1 + m.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Eq(&ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1b"}), + })). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1b"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).MaxTimes(2) + + zone1PublicSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.64.0/18"), + AvailabilityZone: aws.String("us-east-1b"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1b"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.64.0/18"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a02::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1b"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(describeCall) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone1PublicSubnet) + + zone1PrivateSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.0.0/19"), + AvailabilityZone: aws.String("us-east-1b"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a04::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1b"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.0.0/19"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a04::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1b"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone1PublicSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone1PrivateSubnet) + + // zone 2 + m.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{ + ZoneNames: aws.StringSlice([]string{"us-east-1c"}), + }). + Return(&ec2.DescribeAvailabilityZonesOutput{ + AvailabilityZones: []*ec2.AvailabilityZone{ + { + ZoneName: aws.String("us-east-1c"), + ZoneType: aws.String("availability-zone"), + }, + }, + }, nil).AnyTimes() + + zone2PublicSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.128.0/18"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.128.0/18"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a03::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone1PrivateSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone2PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + AssignIpv6AddressOnCreation: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-2"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + m.ModifySubnetAttributeWithContext(context.TODO(), &ec2.ModifySubnetAttributeInput{ + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + SubnetId: aws.String("subnet-1"), + }). + Return(&ec2.ModifySubnetAttributeOutput{}, nil). + After(zone2PublicSubnet) + + zone2PrivateSubnet := m.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String(subnetsVPCID), + CidrBlock: aws.String("10.0.32.0/19"), + AvailabilityZone: aws.String("us-east-1c"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a05::/64"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1c"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })). + Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String(subnetsVPCID), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.32.0/19"), + AssignIpv6AddressOnCreation: aws.Bool(true), + Ipv6CidrBlockAssociationSet: []*ec2.SubnetIpv6CidrBlockAssociation{ + { + AssociationId: aws.String("amazon"), + Ipv6CidrBlock: aws.String("2001:db8:1234:1a05::/64"), + Ipv6CidrBlockState: &ec2.SubnetCidrBlockState{ + State: aws.String(ec2.SubnetCidrBlockStateCodeAssociated), + }, + }, + }, + AvailabilityZone: aws.String("us-east-1c"), + MapPublicIpOnLaunch: aws.Bool(false), + }, + }, nil). + After(zone2PublicSubnet) + + m.WaitUntilSubnetAvailableWithContext(context.TODO(), gomock.Any()). + After(zone2PrivateSubnet) + }, + }, } for _, tc := range testCases {