diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index 36c9b75daf..46106c4bfd 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -307,7 +307,7 @@ func (v *VPCSpec) IsIPv6Enabled() bool { // SubnetSpec configures an AWS Subnet. type SubnetSpec struct { // ID defines a unique identifier to reference this resource. - ID string `json:"id"` + ID string `json:"id,omitempty"` // CidrBlock is the CIDR block to be used when the provider creates a managed VPC. CidrBlock string `json:"cidrBlock,omitempty"` @@ -349,8 +349,6 @@ func (s *SubnetSpec) String() string { } // Subnets is a slice of Subnet. -// +listType=map -// +listMapKey=id type Subnets []SubnetSpec // ToMap returns a map from id to subnet. @@ -373,10 +371,12 @@ func (s Subnets) IDs() []string { } // FindByID returns a single subnet matching the given id or nil. -func (s Subnets) FindByID(id string) *SubnetSpec { - for _, x := range s { +// +// The returned pointer can be used to write back into the original slice. +func (s *Subnets) FindByID(id string) *SubnetSpec { + for i, x := range *s { if x.ID == id { - return &x + return &(*s)[i] // pointer to original structure } } @@ -386,10 +386,12 @@ func (s Subnets) FindByID(id string) *SubnetSpec { // FindEqual returns a subnet spec that is equal to the one passed in. // Two subnets are defined equal to each other if their id is equal // or if they are in the same vpc and the cidr block is the same. -func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec { - for _, x := range s { +// +// The returned pointer can be used to write back into the original slice. +func (s *Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec { + for i, x := range *s { if (spec.ID != "" && x.ID == spec.ID) || (spec.CidrBlock == x.CidrBlock) || (spec.IPv6CidrBlock != "" && spec.IPv6CidrBlock == x.IPv6CidrBlock) { - return &x + return &(*s)[i] // pointer to original structure } } return nil diff --git a/cmd/clusterawsadm/converters/cloudformation.go b/cmd/clusterawsadm/converters/cloudformation.go index d079d1d202..ac7bd4e104 100644 --- a/cmd/clusterawsadm/converters/cloudformation.go +++ b/cmd/clusterawsadm/converters/cloudformation.go @@ -17,6 +17,8 @@ limitations under the License. package converters import ( + "sort" + "github.com/awslabs/goformation/v4/cloudformation/tags" infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" @@ -35,5 +37,8 @@ func MapToCloudFormationTags(src infrav1.Tags) []tags.Tag { cfnTags = append(cfnTags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(cfnTags, func(i, j int) bool { return cfnTags[i].Key < cfnTags[j].Key }) + return cfnTags } diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml index 9612e40ac0..c9c122e8cc 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml @@ -446,13 +446,8 @@ spec: description: Tags is a collection of tags describing the resource. type: object - required: - - id type: object type: array - x-kubernetes-list-map-keys: - - id - x-kubernetes-list-type: map vpc: description: VPC configuration. properties: @@ -1802,13 +1797,8 @@ spec: description: Tags is a collection of tags describing the resource. type: object - required: - - id type: object type: array - x-kubernetes-list-map-keys: - - id - x-kubernetes-list-type: map vpc: description: VPC configuration. properties: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml index 32941aba6e..f4c52c958e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml @@ -1173,13 +1173,8 @@ spec: description: Tags is a collection of tags describing the resource. type: object - required: - - id type: object type: array - x-kubernetes-list-map-keys: - - id - x-kubernetes-list-type: map vpc: description: VPC configuration. properties: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml index 5c6b4e5281..a071d25fd9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml @@ -777,13 +777,8 @@ spec: description: Tags is a collection of tags describing the resource. type: object - required: - - id type: object type: array - x-kubernetes-list-map-keys: - - id - x-kubernetes-list-type: map vpc: description: VPC configuration. properties: diff --git a/controllers/awscluster_controller_test.go b/controllers/awscluster_controller_test.go index cfc3483f6a..833d71347b 100644 --- a/controllers/awscluster_controller_test.go +++ b/controllers/awscluster_controller_test.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/elb" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" "github.com/pkg/errors" @@ -72,8 +73,8 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) { ec2Mock := mocks.NewMockEC2API(mockCtrl) elbMock := mocks.NewMockELBAPI(mockCtrl) expect := func(m *mocks.MockEC2APIMockRecorder, e *mocks.MockELBAPIMockRecorder) { - mockedCreateVPCCalls(m) - mockedCreateSGCalls(false, m) + mockedVPCCallsForExistingVPCAndSubnets(m) + mockedCreateSGCalls(false, "vpc-exists", m) mockedCreateLBCalls(t, e) mockedDescribeInstanceCall(m) } @@ -181,8 +182,8 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) { } expect := func(m *mocks.MockEC2APIMockRecorder, e *mocks.MockELBV2APIMockRecorder) { - mockedCreateVPCCalls(m) - mockedCreateSGCalls(true, m) + mockedVPCCallsForExistingVPCAndSubnets(m) + mockedCreateSGCalls(true, "vpc-exists", m) mockedCreateLBV2Calls(t, e) mockedDescribeInstanceCall(m) } @@ -262,6 +263,112 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) { {conditionType: infrav1.SubnetsReadyCondition, status: corev1.ConditionTrue, severity: "", reason: ""}, }) }) + t.Run("Should successfully reconcile AWSCluster creation with managed VPC", func(t *testing.T) { + g := NewWithT(t) + mockCtrl = gomock.NewController(t) + ec2Mock := mocks.NewMockEC2API(mockCtrl) + elbMock := mocks.NewMockELBAPI(mockCtrl) + expect := func(m *mocks.MockEC2APIMockRecorder, e *mocks.MockELBAPIMockRecorder) { + mockedCallsForMissingEverything(m, e) + mockedCreateSGCalls(false, "vpc-new", m) + mockedDescribeInstanceCall(m) + } + expect(ec2Mock.EXPECT(), elbMock.EXPECT()) + + setup(t) + controllerIdentity := createControllerIdentity(g) + ns, err := testEnv.CreateNamespace(ctx, fmt.Sprintf("integ-test-%s", util.RandomString(5))) + g.Expect(err).To(BeNil()) + + awsCluster := getAWSCluster("test", ns.Name) + awsCluster.Spec.ControlPlaneLoadBalancer = &infrav1.AWSLoadBalancerSpec{ + LoadBalancerType: infrav1.LoadBalancerTypeClassic, + } + + // Make controller manage resources + awsCluster.Spec.NetworkSpec.VPC.ID = "" + awsCluster.Spec.NetworkSpec.Subnets[0].ID = "" + awsCluster.Spec.NetworkSpec.Subnets[1].ID = "" + + // NAT gateway of the public subnet will be accessed by the private subnet in the same zone, + // so use same zone for the 2 test subnets + awsCluster.Spec.NetworkSpec.Subnets[0].AvailabilityZone = "us-east-1a" + awsCluster.Spec.NetworkSpec.Subnets[1].AvailabilityZone = "us-east-1a" + + g.Expect(testEnv.Create(ctx, &awsCluster)).To(Succeed()) + g.Eventually(func() bool { + cluster := &infrav1.AWSCluster{} + key := client.ObjectKey{ + Name: awsCluster.Name, + Namespace: ns.Name, + } + err := testEnv.Get(ctx, key, cluster) + return err == nil + }, 10*time.Second).Should(Equal(true)) + + defer teardown() + defer t.Cleanup(func() { + g.Expect(testEnv.Cleanup(ctx, &awsCluster, controllerIdentity, ns)).To(Succeed()) + }) + + cs, err := getClusterScope(awsCluster) + g.Expect(err).To(BeNil()) + networkSvc := network.NewService(cs) + networkSvc.EC2Client = ec2Mock + reconciler.networkServiceFactory = func(clusterScope scope.ClusterScope) services.NetworkInterface { + return networkSvc + } + + ec2Svc := ec2Service.NewService(cs) + ec2Svc.EC2Client = ec2Mock + reconciler.ec2ServiceFactory = func(scope scope.EC2Scope) services.EC2Interface { + return ec2Svc + } + testSecurityGroupRoles := []infrav1.SecurityGroupRole{ + infrav1.SecurityGroupBastion, + infrav1.SecurityGroupAPIServerLB, + infrav1.SecurityGroupLB, + infrav1.SecurityGroupControlPlane, + infrav1.SecurityGroupNode, + } + sgSvc := securitygroup.NewService(cs, testSecurityGroupRoles) + sgSvc.EC2Client = ec2Mock + + reconciler.securityGroupFactory = func(clusterScope scope.ClusterScope) services.SecurityGroupInterface { + return sgSvc + } + elbSvc := elbService.NewService(cs) + elbSvc.EC2Client = ec2Mock + elbSvc.ELBClient = elbMock + + reconciler.elbServiceFactory = func(elbScope scope.ELBScope) services.ELBInterface { + return elbSvc + } + _, err = reconciler.reconcileNormal(cs) + g.Expect(err).To(BeNil()) + g.Expect(cs.VPC().ID).To(Equal("vpc-new")) + expectAWSClusterConditions(g, cs.AWSCluster, []conditionAssertion{ + {conditionType: infrav1.ClusterSecurityGroupsReadyCondition, status: corev1.ConditionTrue, severity: "", reason: ""}, + {conditionType: infrav1.BastionHostReadyCondition, status: corev1.ConditionTrue, severity: "", reason: ""}, + {conditionType: infrav1.VpcReadyCondition, status: corev1.ConditionTrue, severity: "", reason: ""}, + {conditionType: infrav1.SubnetsReadyCondition, status: corev1.ConditionTrue, severity: "", reason: ""}, + }) + + // Information should get written back into the `ClusterScope` object. Keeping it up to date means that + // reconciliation functionality will always work on the latest-known status of AWS cloud resources. + + // Private subnet + g.Expect(cs.Subnets()[0].ID).To(Equal("subnet-1")) + g.Expect(cs.Subnets()[0].IsPublic).To(Equal(false)) + g.Expect(cs.Subnets()[0].NatGatewayID).To(BeNil()) + g.Expect(cs.Subnets()[0].RouteTableID).To(Equal(aws.String("rtb-1"))) + + // Public subnet + g.Expect(cs.Subnets()[1].ID).To(Equal("subnet-2")) + g.Expect(cs.Subnets()[1].IsPublic).To(Equal(true)) + g.Expect(cs.Subnets()[1].NatGatewayID).To(Equal(aws.String("nat-01"))) + g.Expect(cs.Subnets()[1].RouteTableID).To(Equal(aws.String("rtb-2"))) + }) t.Run("Should fail on AWSCluster reconciliation if VPC limit exceeded", func(t *testing.T) { // Assuming the max VPC limit is 2 and when two VPCs are created, the creation of 3rd VPC throws mocked error from EC2 API @@ -512,7 +619,7 @@ func mockedDeleteInstanceCalls(m *mocks.MockEC2APIMockRecorder) { Return(nil) } -func mockedCreateVPCCalls(m *mocks.MockEC2APIMockRecorder) { +func mockedVPCCallsForExistingVPCAndSubnets(m *mocks.MockEC2APIMockRecorder) { m.CreateTags(gomock.Eq(&ec2.CreateTagsInput{ Resources: aws.StringSlice([]string{"subnet-1"}), Tags: []*ec2.Tag{ @@ -645,6 +752,500 @@ func mockedCreateVPCCalls(m *mocks.MockEC2APIMockRecorder) { }, nil) } +// mockedCallsForMissingEverything mocks most of the AWSCluster reconciliation calls to the AWS API, +// except for what other functions provide (see `mockedCreateSGCalls` and `mockedDescribeInstanceCall`). +func mockedCallsForMissingEverything(m *mocks.MockEC2APIMockRecorder, e *mocks.MockELBAPIMockRecorder) { + m.CreateVpc(gomock.Eq(&ec2.CreateVpcInput{ + AmazonProvidedIpv6CidrBlock: aws.Bool(false), + CidrBlock: aws.String("10.0.0.0/8"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("vpc"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-vpc"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + }, + }, + }, + })).Return(&ec2.CreateVpcOutput{ + Vpc: &ec2.Vpc{ + State: aws.String("available"), + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.0.0.0/8"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-vpc"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + }, + }, + }, nil) + + m.DescribeVpcAttribute(gomock.Eq(&ec2.DescribeVpcAttributeInput{ + VpcId: aws.String("vpc-new"), + Attribute: aws.String("enableDnsHostnames"), + })).Return(&ec2.DescribeVpcAttributeOutput{ + EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(true)}, + }, nil) + + m.DescribeVpcAttribute(gomock.Eq(&ec2.DescribeVpcAttributeInput{ + VpcId: aws.String("vpc-new"), + Attribute: aws.String("enableDnsSupport"), + })).Return(&ec2.DescribeVpcAttributeOutput{ + EnableDnsSupport: &ec2.AttributeBooleanValue{Value: aws.Bool(true)}, + }, nil) + + m.DescribeSubnets(gomock.Eq(&ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("state"), + Values: aws.StringSlice([]string{ec2.VpcStatePending, ec2.VpcStateAvailable}), + }, + { + Name: aws.String("vpc-id"), + Values: aws.StringSlice([]string{"vpc-new"}), + }, + }})).Return(&ec2.DescribeSubnetsOutput{ + Subnets: []*ec2.Subnet{}, + }, nil) + + m.CreateSubnet(gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.0.10.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, + })).Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String("vpc-new"), + SubnetId: aws.String("subnet-1"), + CidrBlock: aws.String("10.0.10.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + MapPublicIpOnLaunch: aws.Bool(false), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-private-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/internal-elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("private"), + }, + }, + }, + }, nil) + + m.WaitUntilSubnetAvailable(gomock.Eq(&ec2.DescribeSubnetsInput{ + SubnetIds: aws.StringSlice([]string{"subnet-1"}), + })).Return(nil) + + m.CreateSubnet(gomock.Eq(&ec2.CreateSubnetInput{ + VpcId: aws.String("vpc-new"), + CidrBlock: aws.String("10.0.11.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("subnet"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, + })).Return(&ec2.CreateSubnetOutput{ + Subnet: &ec2.Subnet{ + VpcId: aws.String("vpc-new"), + SubnetId: aws.String("subnet-2"), + CidrBlock: aws.String("10.0.11.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + MapPublicIpOnLaunch: aws.Bool(false), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-subnet-public-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("shared"), + }, + { + Key: aws.String("kubernetes.io/role/elb"), + Value: aws.String("1"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("public"), + }, + }, + }, + }, nil) + + m.WaitUntilSubnetAvailable(gomock.Eq(&ec2.DescribeSubnetsInput{ + SubnetIds: aws.StringSlice([]string{"subnet-2"}), + })).Return(nil) + + m.ModifySubnetAttribute(gomock.Eq(&ec2.ModifySubnetAttributeInput{ + SubnetId: aws.String("subnet-2"), + MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{ + Value: aws.Bool(true), + }, + })).Return(&ec2.ModifySubnetAttributeOutput{}, nil) + + m.DescribeRouteTables(gomock.Eq(&ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: aws.StringSlice([]string{"vpc-new"}), + }, + { + Name: aws.String("tag-key"), + Values: aws.StringSlice([]string{"sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"}), + }, + }})).Return(&ec2.DescribeRouteTablesOutput{ + RouteTables: []*ec2.RouteTable{ + { + Routes: []*ec2.Route{ + { + GatewayId: aws.String("igw-12345"), + }, + }, + }, + }, + }, nil).MinTimes(1).MaxTimes(2) + + m.DescribeInternetGateways(gomock.Eq(&ec2.DescribeInternetGatewaysInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("attachment.vpc-id"), + Values: aws.StringSlice([]string{"vpc-new"}), + }, + }, + })).Return(&ec2.DescribeInternetGatewaysOutput{ + InternetGateways: []*ec2.InternetGateway{}, + }, nil) + + m.CreateInternetGateway(gomock.AssignableToTypeOf(&ec2.CreateInternetGatewayInput{})). + Return(&ec2.CreateInternetGatewayOutput{ + InternetGateway: &ec2.InternetGateway{ + InternetGatewayId: aws.String("igw-1"), + Tags: []*ec2.Tag{ + { + Key: aws.String(infrav1.ClusterTagKey("test-cluster")), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-igw"), + }, + }, + }, + }, nil) + + m.AttachInternetGateway(gomock.Eq(&ec2.AttachInternetGatewayInput{ + InternetGatewayId: aws.String("igw-1"), + VpcId: aws.String("vpc-new"), + })). + Return(&ec2.AttachInternetGatewayOutput{}, nil) + + m.DescribeNatGatewaysPages(gomock.Eq(&ec2.DescribeNatGatewaysInput{ + Filter: []*ec2.Filter{ + { + Name: aws.String("vpc-id"), + Values: []*string{aws.String("vpc-new")}, + }, + { + Name: aws.String("state"), + Values: aws.StringSlice([]string{ec2.VpcStatePending, ec2.VpcStateAvailable}), + }, + }}), gomock.Any()).Return(nil).MinTimes(1).MaxTimes(2) + + m.DescribeAddresses(gomock.Eq(&ec2.DescribeAddressesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("tag-key"), + Values: aws.StringSlice([]string{"sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"}), + }, + { + Name: aws.String("tag:sigs.k8s.io/cluster-api-provider-aws/role"), + Values: aws.StringSlice([]string{"apiserver"}), + }, + }, + })).Return(&ec2.DescribeAddressesOutput{ + Addresses: []*ec2.Address{}, + }, nil) + + m.AllocateAddress(gomock.Eq(&ec2.AllocateAddressInput{ + Domain: aws.String("vpc"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("elastic-ip"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-eip-apiserver"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("apiserver"), + }, + }, + }, + }, + })).Return(&ec2.AllocateAddressOutput{ + AllocationId: aws.String("1234"), + }, nil) + + m.CreateNatGateway(gomock.Eq(&ec2.CreateNatGatewayInput{ + AllocationId: aws.String("1234"), + SubnetId: aws.String("subnet-2"), + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("natgateway"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-nat"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + }, + }, + }, + })).Return(&ec2.CreateNatGatewayOutput{ + NatGateway: &ec2.NatGateway{ + NatGatewayId: aws.String("nat-01"), + SubnetId: aws.String("subnet-2"), + }, + }, nil) + + m.WaitUntilNatGatewayAvailable(&ec2.DescribeNatGatewaysInput{ + NatGatewayIds: []*string{aws.String("nat-01")}, + }).Return(nil) + + m.CreateRouteTable(gomock.Eq(&ec2.CreateRouteTableInput{ + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("route-table"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-rt-private-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + }, + }, + }, + VpcId: aws.String("vpc-new"), + })).Return(&ec2.CreateRouteTableOutput{ + RouteTable: &ec2.RouteTable{ + RouteTableId: aws.String("rtb-1"), + }, + }, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationCidrBlock: aws.String("0.0.0.0/0"), + NatGatewayId: aws.String("nat-01"), + RouteTableId: aws.String("rtb-1"), + })).Return(&ec2.CreateRouteOutput{}, nil) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rtb-1"), + SubnetId: aws.String("subnet-1"), + })).Return(&ec2.AssociateRouteTableOutput{}, nil) + + m.CreateRouteTable(gomock.Eq(&ec2.CreateRouteTableInput{ + TagSpecifications: []*ec2.TagSpecification{ + { + ResourceType: aws.String("route-table"), + Tags: []*ec2.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-rt-public-us-east-1a"), + }, + { + Key: aws.String("kubernetes.io/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("common"), + }, + }, + }, + }, + VpcId: aws.String("vpc-new"), + })).Return(&ec2.CreateRouteTableOutput{ + RouteTable: &ec2.RouteTable{ + RouteTableId: aws.String("rtb-2"), + }, + }, nil) + + m.CreateRoute(gomock.Eq(&ec2.CreateRouteInput{ + DestinationCidrBlock: aws.String("0.0.0.0/0"), + GatewayId: aws.String("igw-1"), + RouteTableId: aws.String("rtb-2"), + })).Return(&ec2.CreateRouteOutput{}, nil) + + m.AssociateRouteTable(gomock.Eq(&ec2.AssociateRouteTableInput{ + RouteTableId: aws.String("rtb-2"), + SubnetId: aws.String("subnet-2"), + })).Return(&ec2.AssociateRouteTableOutput{}, nil) + + e.DescribeLoadBalancers(gomock.Eq(&elb.DescribeLoadBalancersInput{ + LoadBalancerNames: aws.StringSlice([]string{"test-cluster-apiserver"}), + })).Return(&elb.DescribeLoadBalancersOutput{ + LoadBalancerDescriptions: []*elb.LoadBalancerDescription{}, + }, nil) + + e.CreateLoadBalancer(gomock.Eq(&elb.CreateLoadBalancerInput{ + Listeners: []*elb.Listener{ + { + InstancePort: aws.Int64(6443), + InstanceProtocol: aws.String("TCP"), + LoadBalancerPort: aws.Int64(6443), + Protocol: aws.String("TCP"), + }, + }, + LoadBalancerName: aws.String("test-cluster-apiserver"), + Scheme: aws.String("internet-facing"), + SecurityGroups: aws.StringSlice([]string{"sg-apiserver-lb"}), + Subnets: aws.StringSlice([]string{"subnet-2"}), + Tags: []*elb.Tag{ + { + Key: aws.String("Name"), + Value: aws.String("test-cluster-apiserver"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/cluster/test-cluster"), + Value: aws.String("owned"), + }, + { + Key: aws.String("sigs.k8s.io/cluster-api-provider-aws/role"), + Value: aws.String("apiserver"), + }, + }, + })).Return(&elb.CreateLoadBalancerOutput{ + DNSName: aws.String("unittest24.de"), + }, nil) + + e.ConfigureHealthCheck(gomock.Eq(&elb.ConfigureHealthCheckInput{ + LoadBalancerName: aws.String("test-cluster-apiserver"), + HealthCheck: &elb.HealthCheck{ + Target: aws.String("SSL:6443"), + Interval: aws.Int64(10), + Timeout: aws.Int64(5), + HealthyThreshold: aws.Int64(5), + UnhealthyThreshold: aws.Int64(3), + }, + })).Return(&elb.ConfigureHealthCheckOutput{}, nil) +} + func mockedCreateMaximumVPCCalls(m *mocks.MockEC2APIMockRecorder) { m.CreateVpc(gomock.AssignableToTypeOf(&ec2.CreateVpcInput{})).Return(nil, errors.New("The maximum number of VPCs has been reached")) } @@ -846,12 +1447,12 @@ func mockedDeleteVPCCalls(m *mocks.MockEC2APIMockRecorder) { })) } -func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { +func mockedCreateSGCalls(recordLBV2 bool, vpcId string, m *mocks.MockEC2APIMockRecorder) { m.DescribeSecurityGroups(gomock.Eq(&ec2.DescribeSecurityGroupsInput{ Filters: []*ec2.Filter{ { Name: aws.String("vpc-id"), - Values: aws.StringSlice([]string{"vpc-exists"}), + Values: aws.StringSlice([]string{vpcId}), }, { Name: aws.String("tag-key"), @@ -868,7 +1469,7 @@ func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { }, }, nil) m.CreateSecurityGroup(gomock.Eq(&ec2.CreateSecurityGroupInput{ - VpcId: aws.String("vpc-exists"), + VpcId: aws.String(vpcId), GroupName: aws.String("test-cluster-bastion"), Description: aws.String("Kubernetes cluster test-cluster: bastion"), TagSpecifications: []*ec2.TagSpecification{ @@ -893,7 +1494,7 @@ func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { })). Return(&ec2.CreateSecurityGroupOutput{GroupId: aws.String("sg-bastion")}, nil) m.CreateSecurityGroup(gomock.Eq(&ec2.CreateSecurityGroupInput{ - VpcId: aws.String("vpc-exists"), + VpcId: aws.String(vpcId), GroupName: aws.String("test-cluster-apiserver-lb"), Description: aws.String("Kubernetes cluster test-cluster: apiserver-lb"), TagSpecifications: []*ec2.TagSpecification{ @@ -918,7 +1519,7 @@ func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { })). Return(&ec2.CreateSecurityGroupOutput{GroupId: aws.String("sg-apiserver-lb")}, nil) m.CreateSecurityGroup(gomock.Eq(&ec2.CreateSecurityGroupInput{ - VpcId: aws.String("vpc-exists"), + VpcId: aws.String(vpcId), GroupName: aws.String("test-cluster-lb"), Description: aws.String("Kubernetes cluster test-cluster: lb"), TagSpecifications: []*ec2.TagSpecification{ @@ -947,7 +1548,7 @@ func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { })). Return(&ec2.CreateSecurityGroupOutput{GroupId: aws.String("sg-lb")}, nil) securityGroupControl := m.CreateSecurityGroup(gomock.Eq(&ec2.CreateSecurityGroupInput{ - VpcId: aws.String("vpc-exists"), + VpcId: aws.String(vpcId), GroupName: aws.String("test-cluster-controlplane"), Description: aws.String("Kubernetes cluster test-cluster: controlplane"), TagSpecifications: []*ec2.TagSpecification{ @@ -972,7 +1573,7 @@ func mockedCreateSGCalls(recordLBV2 bool, m *mocks.MockEC2APIMockRecorder) { })). Return(&ec2.CreateSecurityGroupOutput{GroupId: aws.String("sg-controlplane")}, nil) securityGroupNode := m.CreateSecurityGroup(gomock.Eq(&ec2.CreateSecurityGroupInput{ - VpcId: aws.String("vpc-exists"), + VpcId: aws.String(vpcId), GroupName: aws.String("test-cluster-node"), Description: aws.String("Kubernetes cluster test-cluster: node"), TagSpecifications: []*ec2.TagSpecification{ diff --git a/pkg/cloud/converters/tags.go b/pkg/cloud/converters/tags.go index 7bec800206..c46c412d7c 100644 --- a/pkg/cloud/converters/tags.go +++ b/pkg/cloud/converters/tags.go @@ -17,6 +17,8 @@ limitations under the License. package converters import ( + "sort" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" @@ -64,6 +66,9 @@ func MapToTags(src infrav1.Tags) []*ec2.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -102,6 +107,9 @@ func MapToELBTags(src infrav1.Tags) []*elb.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -118,6 +126,9 @@ func MapToV2Tags(src infrav1.Tags) []*elbv2.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -134,6 +145,9 @@ func MapToSecretsManagerTags(src infrav1.Tags) []*secretsmanager.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -150,6 +164,9 @@ func MapToSSMTags(src infrav1.Tags) []*ssm.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -166,6 +183,9 @@ func MapToIAMTags(src infrav1.Tags) []*iam.Tag { tags = append(tags, tag) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } diff --git a/pkg/cloud/services/autoscaling/autoscalinggroup.go b/pkg/cloud/services/autoscaling/autoscalinggroup.go index b1a08c6c61..7b88a78ecb 100644 --- a/pkg/cloud/services/autoscaling/autoscalinggroup.go +++ b/pkg/cloud/services/autoscaling/autoscalinggroup.go @@ -18,6 +18,7 @@ package asg import ( "fmt" + "sort" "strings" "github.com/aws/aws-sdk-go/aws" @@ -415,6 +416,9 @@ func BuildTagsFromMap(asgName string, inTags map[string]string) []*autoscaling.T }) } + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } @@ -493,6 +497,10 @@ func mapToTags(input map[string]string, resourceID *string) []*autoscaling.Tag { Value: aws.String(v), }) } + + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } diff --git a/pkg/cloud/services/ec2/launchtemplate.go b/pkg/cloud/services/ec2/launchtemplate.go index c547ad08a4..19cad31137 100644 --- a/pkg/cloud/services/ec2/launchtemplate.go +++ b/pkg/cloud/services/ec2/launchtemplate.go @@ -806,6 +806,8 @@ func (s *Service) buildLaunchTemplateTagSpecificationRequest(scope scope.LaunchT Value: aws.String(value), }) } + // Sort so that unit tests can expect a stable order + sort.Slice(spec.Tags, func(i, j int) bool { return *spec.Tags[i].Key < *spec.Tags[j].Key }) tagSpecifications = append(tagSpecifications, spec) // tag EBS volumes @@ -816,6 +818,8 @@ func (s *Service) buildLaunchTemplateTagSpecificationRequest(scope scope.LaunchT Value: aws.String(value), }) } + // Sort so that unit tests can expect a stable order + sort.Slice(spec.Tags, func(i, j int) bool { return *spec.Tags[i].Key < *spec.Tags[j].Key }) tagSpecifications = append(tagSpecifications, spec) } return tagSpecifications diff --git a/pkg/cloud/services/eks/iam/iam.go b/pkg/cloud/services/eks/iam/iam.go index b4e77b3864..7e63c818b4 100644 --- a/pkg/cloud/services/eks/iam/iam.go +++ b/pkg/cloud/services/eks/iam/iam.go @@ -23,6 +23,7 @@ import ( "encoding/json" "net/http" "net/url" + "sort" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/eks" @@ -174,6 +175,10 @@ func RoleTags(key string, additionalTags infrav1.Tags) []*iam.Tag { Value: aws.String(v), }) } + + // Sort so that unit tests can expect a stable order + sort.Slice(tags, func(i, j int) bool { return *tags[i].Key < *tags[j].Key }) + return tags } diff --git a/pkg/cloud/services/elb/loadbalancer.go b/pkg/cloud/services/elb/loadbalancer.go index b318defbd0..4ad5cdc0a2 100644 --- a/pkg/cloud/services/elb/loadbalancer.go +++ b/pkg/cloud/services/elb/loadbalancer.go @@ -698,20 +698,18 @@ func (s *Service) RegisterInstanceWithAPIServerELB(i *infrav1.Instance) error { } // Validate that the subnets associated with the load balancer has the instance AZ. - subnet := s.scope.Subnets().FindByID(i.SubnetID) - if subnet == nil { + subnets := s.scope.Subnets() + instanceSubnet := subnets.FindByID(i.SubnetID) + if instanceSubnet == nil { return errors.Errorf("failed to attach load balancer subnets, could not find subnet %q description in AWSCluster", i.SubnetID) } - instanceAZ := subnet.AvailabilityZone + instanceAZ := instanceSubnet.AvailabilityZone - var subnets infrav1.Subnets if s.scope.ControlPlaneLoadBalancer() != nil && len(s.scope.ControlPlaneLoadBalancer().Subnets) > 0 { subnets, err = s.getControlPlaneLoadBalancerSubnets() if err != nil { return err } - } else { - subnets = s.scope.Subnets() } found := false diff --git a/pkg/cloud/services/network/natgateways.go b/pkg/cloud/services/network/natgateways.go index 8e42c184c7..656d34a7ab 100644 --- a/pkg/cloud/services/network/natgateways.go +++ b/pkg/cloud/services/network/natgateways.go @@ -107,8 +107,12 @@ func (s *Service) reconcileNatGateways() error { } ngws, err := s.createNatGateways(subnetIDs) + subnets := s.scope.Subnets() + defer func() { + s.scope.SetSubnets(subnets) + }() for _, ng := range ngws { - subnet := s.scope.Subnets().FindByID(*ng.SubnetId) + subnet := subnets.FindByID(*ng.SubnetId) subnet.NatGatewayID = ng.NatGatewayId } diff --git a/pkg/cloud/services/network/routetables.go b/pkg/cloud/services/network/routetables.go index c94acb7634..24a8325365 100644 --- a/pkg/cloud/services/network/routetables.go +++ b/pkg/cloud/services/network/routetables.go @@ -52,8 +52,12 @@ func (s *Service) reconcileRouteTables() error { } subnets := s.scope.Subnets() + defer func() { + s.scope.SetSubnets(subnets) + }() + for i := range subnets { - sn := subnets[i] + sn := &subnets[i] // We need to compile the minimum routes for this subnet first, so we can compare it or create them. var routes []*ec2.Route if sn.IsPublic { @@ -65,7 +69,7 @@ func (s *Service) reconcileRouteTables() error { routes = append(routes, s.getGatewayPublicIPv6Route()) } } else { - natGatewayID, err := s.getNatGatewayForSubnet(&sn) + natGatewayID, err := s.getNatGatewayForSubnet(sn) if err != nil { return err } diff --git a/pkg/cloud/services/network/subnets.go b/pkg/cloud/services/network/subnets.go index 5b9f193757..595db393b8 100644 --- a/pkg/cloud/services/network/subnets.go +++ b/pkg/cloud/services/network/subnets.go @@ -141,7 +141,7 @@ func (s *Service) reconcileSubnets() error { } else if unmanagedVPC { // If there is no existing subnet and we have an umanaged vpc report an error record.Warnf(s.scope.InfraCluster(), "FailedMatchSubnet", "Using unmanaged VPC and failed to find existing subnet for specified subnet id %d, cidr %q", sub.ID, sub.CidrBlock) - return errors.New(fmt.Errorf("usign unmanaged vpc and subnet %s (cidr %s) specified but it doesn't exist in vpc %s", sub.ID, sub.CidrBlock, s.scope.VPC().ID).Error()) + return errors.New(fmt.Errorf("using unmanaged vpc and subnet %s (cidr %s) specified but it doesn't exist in vpc %s", sub.ID, sub.CidrBlock, s.scope.VPC().ID).Error()) } } diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 503647f1db..32e62fb488 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -104,6 +104,7 @@ providers: - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-limit-az.yaml" - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-machine-pool.yaml" - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-md-remediation.yaml" + - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-multi-az.yaml" - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-nested-multitenancy.yaml" - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-remote-management-cluster.yaml" - sourcePath: "./infrastructure-aws/withoutclusterclass/generated/cluster-template-simple-multitenancy.yaml" diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/kustomization.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/kustomization.yaml new file mode 100644 index 0000000000..9cc991fcd9 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - ../default +patchesStrategicMerge: + - patches/multi-az.yaml diff --git a/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/patches/multi-az.yaml b/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/patches/multi-az.yaml new file mode 100644 index 0000000000..a4a158fbe1 --- /dev/null +++ b/test/e2e/data/infrastructure-aws/kustomize_sources/multi-az/patches/multi-az.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + network: + subnets: + - availabilityZone: "${AWS_AVAILABILITY_ZONE_1}" + cidrBlock: "10.0.0.0/24" + - availabilityZone: "${AWS_AVAILABILITY_ZONE_1}" + cidrBlock: "10.0.1.0/24" + isPublic: true + - availabilityZone: "${AWS_AVAILABILITY_ZONE_2}" + cidrBlock: "10.0.2.0/24" + - availabilityZone: "${AWS_AVAILABILITY_ZONE_2}" + cidrBlock: "10.0.3.0/24" + isPublic: true diff --git a/test/e2e/shared/defaults.go b/test/e2e/shared/defaults.go index 5760ce1cfb..7335b36a56 100644 --- a/test/e2e/shared/defaults.go +++ b/test/e2e/shared/defaults.go @@ -49,6 +49,7 @@ const ( AwsNodeMachineType = "AWS_NODE_MACHINE_TYPE" AwsAvailabilityZone1 = "AWS_AVAILABILITY_ZONE_1" AwsAvailabilityZone2 = "AWS_AVAILABILITY_ZONE_2" + MultiAzFlavor = "multi-az" LimitAzFlavor = "limit-az" SpotInstancesFlavor = "spot-instances" SSMFlavor = "ssm" diff --git a/test/e2e/suites/unmanaged/helpers_test.go b/test/e2e/suites/unmanaged/helpers_test.go index af74437730..c77cca3ad1 100644 --- a/test/e2e/suites/unmanaged/helpers_test.go +++ b/test/e2e/suites/unmanaged/helpers_test.go @@ -388,6 +388,35 @@ func getEvents(namespace string) *corev1.EventList { return eventsList } +func getSubnetID(filterKey, filterValue, clusterName string) *string { + var subnetOutput *ec2.DescribeSubnetsOutput + var err error + + ec2Client := ec2.New(e2eCtx.AWSSession) + subnetInput := &ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String(filterKey), + Values: []*string{ + aws.String(filterValue), + }, + }, + { + Name: aws.String("tag-key"), + Values: aws.StringSlice([]string{"sigs.k8s.io/cluster-api-provider-aws/cluster/" + clusterName}), + }, + }, + } + + Eventually(func() int { + subnetOutput, err = ec2Client.DescribeSubnets(subnetInput) + Expect(err).NotTo(HaveOccurred()) + return len(subnetOutput.Subnets) + }, e2eCtx.E2EConfig.GetIntervals("", "wait-infra-subnets")...).Should(Equal(1)) + + return subnetOutput.Subnets[0].SubnetId +} + func getVolumeIds(info statefulSetInfo, k8sclient crclient.Client) []*string { ginkgo.By("Retrieving IDs of dynamically provisioned volumes.") statefulset := &appsv1.StatefulSet{} diff --git a/test/e2e/suites/unmanaged/unmanaged_functional_test.go b/test/e2e/suites/unmanaged/unmanaged_functional_test.go index 9e5b7f727f..7718827312 100644 --- a/test/e2e/suites/unmanaged/unmanaged_functional_test.go +++ b/test/e2e/suites/unmanaged/unmanaged_functional_test.go @@ -22,6 +22,7 @@ package unmanaged import ( "context" "fmt" + "os" "path/filepath" "strings" "time" @@ -548,6 +549,51 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() { }) }) + ginkgo.Describe("Workload cluster in multiple AZs", func() { + ginkgo.It("It should be creatable and deletable", func() { + specName := "functional-test-multi-az" + requiredResources = &shared.TestResource{EC2Normal: 3 * e2eCtx.Settings.InstanceVCPU, IGW: 1, NGW: 1, VPC: 1, ClassicLB: 1, EIP: 3} + requiredResources.WriteRequestedResources(e2eCtx, specName) + Expect(shared.AcquireResources(requiredResources, config.GinkgoConfig.ParallelNode, flock.New(shared.ResourceQuotaFilePath))).To(Succeed()) + defer shared.ReleaseResources(requiredResources, config.GinkgoConfig.ParallelNode, flock.New(shared.ResourceQuotaFilePath)) + namespace := shared.SetupSpecNamespace(ctx, specName, e2eCtx) + defer shared.DumpSpecResourcesAndCleanup(ctx, "", namespace, e2eCtx) + ginkgo.By("Creating a cluster") + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + configCluster := defaultConfigCluster(clusterName, namespace.Name) + configCluster.ControlPlaneMachineCount = pointer.Int64Ptr(3) + configCluster.Flavor = shared.MultiAzFlavor + cluster, _, _ := createCluster(ctx, configCluster, result) + + ginkgo.By("Adding worker nodes to additional subnets") + mdName1 := clusterName + "-md-1" + mdName2 := clusterName + "-md-2" + md1 := makeMachineDeployment(namespace.Name, mdName1, clusterName, 1) + md2 := makeMachineDeployment(namespace.Name, mdName2, clusterName, 1) + az1 := os.Getenv(shared.AwsAvailabilityZone1) + az2 := os.Getenv(shared.AwsAvailabilityZone2) + + // private CIDRs set in cluster-template-multi-az.yaml. + framework.CreateMachineDeployment(ctx, framework.CreateMachineDeploymentInput{ + Creator: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + MachineDeployment: md1, + BootstrapConfigTemplate: makeJoinBootstrapConfigTemplate(namespace.Name, mdName1), + InfraMachineTemplate: makeAWSMachineTemplate(namespace.Name, mdName1, e2eCtx.E2EConfig.GetVariable(shared.AwsNodeMachineType), pointer.StringPtr(az1), getSubnetID("cidr-block", "10.0.0.0/24", clusterName)), + }) + framework.CreateMachineDeployment(ctx, framework.CreateMachineDeploymentInput{ + Creator: e2eCtx.Environment.BootstrapClusterProxy.GetClient(), + MachineDeployment: md2, + BootstrapConfigTemplate: makeJoinBootstrapConfigTemplate(namespace.Name, mdName2), + InfraMachineTemplate: makeAWSMachineTemplate(namespace.Name, mdName2, e2eCtx.E2EConfig.GetVariable(shared.AwsNodeMachineType), pointer.StringPtr(az2), getSubnetID("cidr-block", "10.0.2.0/24", clusterName)), + }) + + ginkgo.By("Waiting for new worker nodes to become ready") + k8sClient := e2eCtx.Environment.BootstrapClusterProxy.GetClient() + framework.WaitForMachineDeploymentNodesToExist(ctx, framework.WaitForMachineDeploymentNodesToExistInput{Lister: k8sClient, Cluster: cluster, MachineDeployment: md1}, e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes")...) + framework.WaitForMachineDeploymentNodesToExist(ctx, framework.WaitForMachineDeploymentNodesToExistInput{Lister: k8sClient, Cluster: cluster, MachineDeployment: md2}, e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes")...) + }) + }) + // TODO @randomvariable: Await more resources ginkgo.PDescribe("Multiple workload clusters", func() { ginkgo.Context("in different namespaces with machine failures", func() {