✨ edge subnets/gateway: add gateway routing for Local Zones

✨ edge subnets/routes: supporting custom routes for Local Zones

Isolate the route table lookup into dedicated methods for private and
public subnets to allow more complex requirements for edge zones, as
well introduce unit tests for each scenario to cover edge cases.

There is no change for private and public subnets for regular
zones (standard flow), and the routes will be assigned accordainly
the existing flow: private subnets uses nat gateways per public zone,
and internet gateway for public zones's tables.

For private and public subnets in edge zones, the following changes is
introduced according to each rule:

General:

- IPv6 subnets is not be supported in AWS Local Zones,
  zone, consequently no ip6 routes will be created
- nat gateways is not supported, default gateway's route for private
  subnets will use nat gateways from the zones in the Region
(availability-zone's zone type)
- one route table by zone's role by zone (standard flow)

Private tables for Local Zones:
- default route's gateways is assigned using nat gateway created in
  the region (availability-zones).

Public tables for Local Zones:
- default route's gateway is assigned using internet gateway

The changes in the standard flow (without edge subnets' support) was
isolated in the PR kubernetes-sigs#4900

✨ edge subnets/nat-gw: support private routing in Local Zones

Introduce the support to lookup a nat gateway for edge zones when
creating private subnets.

Currently CAPA requires a NAT Gateway in the public subnet for each zone
which requires private subnets to define default nat gateway in the
private route table for each zone.

NAT Gateway resource isn't globally supported by Local Zones, thus
private subnets in Local Zones are created with default route gateway
using a nat gateway selected in the Region (regular availability zones)
based in the Parent Zone* for the edge subnet.

*each edge zone is "tied" to a zone named "Parent Zone", a zone type
availability-zone (regular zones) in the region.

pkg/cloud/services/network/natgateways.go

@@ -19,6 +19,7 @@ package network
 import (
 	"context"
 	"fmt"
+	"sort"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -321,23 +322,55 @@ func (s *Service) deleteNatGateway(id string) error {
 	return nil
 }
 
+// getNatGatewayForSubnet return the nat gateway for private subnets.
+// NAT gateways in edge zones (Local Zones) are not globally supported,
+// private subnets in those locations uses Nat Gateways from the
+// Parent Zone or, when not available, the first zone in the Region.
 func (s *Service) getNatGatewayForSubnet(sn *infrav1.SubnetSpec) (string, error) {
 	if sn.IsPublic {
 		return "", errors.Errorf("cannot get NAT gateway for a public subnet, got id %q", sn.GetResourceID())
 	}
 
-	azGateways := make(map[string][]string)
+	// Check if public edge subnet in the edge zone has nat gateway
+	azGateways := make(map[string]string)
+	azNames := []string{}
 	for _, psn := range s.scope.Subnets().FilterPublic() {
 		if psn.NatGatewayID == nil {
 			continue
 		}
-
-		azGateways[psn.AvailabilityZone] = append(azGateways[psn.AvailabilityZone], *psn.NatGatewayID)
+		if _, ok := azGateways[psn.AvailabilityZone]; !ok {
+			azGateways[psn.AvailabilityZone] = *psn.NatGatewayID
+			azNames = append(azNames, psn.AvailabilityZone)
+		}
 	}
 
 	if gws, ok := azGateways[sn.AvailabilityZone]; ok && len(gws) > 0 {
-		return gws[0], nil
+		return gws, nil
+	}
+
+	// return error when no gateway found for regular zones, availability-zone zone type.
+	if !sn.IsEdge() {
+		return "", errors.Errorf("no nat gateways available in %q for private subnet %q", sn.AvailabilityZone, sn.GetResourceID())
+	}
+
+	// edge zones only: trying to find nat gateway for Local or Wavelength zone based in the zone type.
+
+	// Check if the parent zone public subnet has nat gateway
+	if sn.ParentZoneName != nil {
+		if gws, ok := azGateways[aws.StringValue(sn.ParentZoneName)]; ok && len(gws) > 0 {
+			return gws, nil
+		}
+	}
+
+	// Get the first public subnet's nat gateway available
+	sort.Strings(azNames)
+	for _, zone := range azNames {
+		gw := azGateways[zone]
+		if len(gw) > 0 {
+			s.scope.Debug("Assigning route table", "table ID", gw, "source zone", zone, "target zone", sn.AvailabilityZone)
+			return gw, nil
+		}
 	}
 
-	return "", errors.Errorf("no nat gateways available in %q for private subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways)
+	return "", errors.Errorf("no nat gateways available in %q for private edge subnet %q, current state: %+v", sn.AvailabilityZone, sn.GetResourceID(), azGateways)
 }

pkg/cloud/services/network/natgateways_test.go

@@ -27,6 +27,7 @@ import (
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
@@ -728,3 +729,262 @@ var mockDescribeNatGatewaysOutput = func(ctx context.Context, _, y interface{},
 		SubnetId:     aws.String("subnet-1"),
 	}}}, true)
 }
+
+func TestGetdNatGatewayForEdgeSubnet(t *testing.T) {
+	subnetsSpec := infrav1.Subnets{
+		{
+			ID:               "subnet-az-1x-private",
+			AvailabilityZone: "us-east-1x",
+			IsPublic:         false,
+		},
+		{
+			ID:               "subnet-az-1x-public",
+			AvailabilityZone: "us-east-1x",
+			IsPublic:         true,
+			NatGatewayID:     aws.String("natgw-az-1b-last"),
+		},
+		{
+			ID:               "subnet-az-1a-private",
+			AvailabilityZone: "us-east-1a",
+			IsPublic:         false,
+		},
+		{
+			ID:               "subnet-az-1a-public",
+			AvailabilityZone: "us-east-1a",
+			IsPublic:         true,
+			NatGatewayID:     aws.String("natgw-az-1b-first"),
+		},
+		{
+			ID:               "subnet-az-1b-private",
+			AvailabilityZone: "us-east-1b",
+			IsPublic:         false,
+		},
+		{
+			ID:               "subnet-az-1b-public",
+			AvailabilityZone: "us-east-1b",
+			IsPublic:         true,
+			NatGatewayID:     aws.String("natgw-az-1b-second"),
+		},
+		{
+			ID:               "subnet-az-1p-private",
+			AvailabilityZone: "us-east-1p",
+			IsPublic:         false,
+		},
+	}
+
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	testCases := []struct {
+		name             string
+		spec             infrav1.Subnets
+		input            infrav1.SubnetSpec
+		expect           string
+		expectErr        bool
+		expectErrMessage string
+	}{
+		{
+			name: "zone availability-zone, valid nat gateway",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-az-1b-private",
+				AvailabilityZone: "us-east-1b",
+				IsPublic:         false,
+			},
+			expect: "natgw-az-1b-second",
+		},
+		{
+			name: "zone availability-zone, valid nat gateway",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-az-1a-private",
+				AvailabilityZone: "us-east-1a",
+				IsPublic:         false,
+			},
+			expect: "natgw-az-1b-first",
+		},
+		{
+			name: "zone availability-zone, valid nat gateway",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-az-1x-private",
+				AvailabilityZone: "us-east-1x",
+				IsPublic:         false,
+			},
+			expect: "natgw-az-1b-last",
+		},
+		{
+			name: "zone local-zone, valid nat gateway from parent",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-nyc1a-private",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         false,
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+				ParentZoneName:   aws.String("us-east-1a"),
+			},
+			expect: "natgw-az-1b-first",
+		},
+		{
+			name: "zone local-zone, valid nat gateway from parent",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-nyc1a-private",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         false,
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+				ParentZoneName:   aws.String("us-east-1x"),
+			},
+			expect: "natgw-az-1b-last",
+		},
+		{
+			name: "zone local-zone, valid nat gateway from fallback",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-nyc1a-private",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         false,
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+				ParentZoneName:   aws.String("us-east-1-notAvailable"),
+			},
+			expect: "natgw-az-1b-first",
+		},
+		{
+			name: "edge zones without NAT GW support, no public subnet and NAT Gateway for the parent zone, return first nat gateway available",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-7",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+			},
+			expect: "natgw-az-1b-first",
+		},
+		{
+			name: "edge zones without NAT GW support, no public subnet and NAT Gateway for the parent zone, return first nat gateway available",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-7",
+				CidrBlock:        "10.0.10.0/24",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+				ParentZoneName:   aws.String("us-east-1-notFound"),
+			},
+			expect: "natgw-az-1b-first",
+		},
+		{
+			name: "edge zones without NAT GW support, valid public subnet and NAT Gateway for the parent zone, return parent's zone nat gateway",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-7",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+				ParentZoneName:   aws.String("us-east-1b"),
+			},
+			expect: "natgw-az-1b-second",
+		},
+		// errors
+		{
+			name: "error if the subnet is public",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-az-1-public",
+				AvailabilityZone: "us-east-1a",
+				IsPublic:         true,
+			},
+			expectErr:        true,
+			expectErrMessage: `cannot get NAT gateway for a public subnet, got id "subnet-az-1-public"`,
+		},
+		{
+			name: "error if the subnet is public",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-1-public",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         true,
+			},
+			expectErr:        true,
+			expectErrMessage: `cannot get NAT gateway for a public subnet, got id "subnet-lz-1-public"`,
+		},
+		{
+			name: "error if there are no nat gateways available in the subnets",
+			spec: infrav1.Subnets{},
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-az-1-private",
+				AvailabilityZone: "us-east-1p",
+				IsPublic:         false,
+			},
+			expectErr:        true,
+			expectErrMessage: `no nat gateways available in "us-east-1p" for private subnet "subnet-az-1-private"`,
+		},
+		{
+			name: "error if there are no nat gateways available in the subnets",
+			spec: infrav1.Subnets{},
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-1",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         false,
+				ZoneType:         ptr.To(infrav1.ZoneTypeLocalZone),
+			},
+			expectErr:        true,
+			expectErrMessage: `no nat gateways available in "us-east-1-nyc-1a" for private edge subnet "subnet-lz-1", current state: map[]`,
+		},
+		{
+			name: "error if the subnet is public",
+			input: infrav1.SubnetSpec{
+				ID:               "subnet-lz-1",
+				AvailabilityZone: "us-east-1-nyc-1a",
+				IsPublic:         true,
+			},
+			expectErr:        true,
+			expectErrMessage: `cannot get NAT gateway for a public subnet, got id "subnet-lz-1"`,
+		},
+	}
+
+	for idx, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			subnets := subnetsSpec
+			if tc.spec != nil {
+				subnets = tc.spec
+			}
+			scheme := runtime.NewScheme()
+			_ = infrav1.AddToScheme(scheme)
+			awsCluster := &infrav1.AWSCluster{
+				ObjectMeta: metav1.ObjectMeta{Name: "test"},
+				Spec: infrav1.AWSClusterSpec{
+					NetworkSpec: infrav1.NetworkSpec{
+						VPC: infrav1.VPCSpec{
+							ID: subnetsVPCID,
+							Tags: infrav1.Tags{
+								infrav1.ClusterTagKey("test-cluster"): "owned",
+							},
+						},
+						Subnets: subnets,
+					},
+				},
+			}
+
+			client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(awsCluster).WithStatusSubresource(awsCluster).Build()
+
+			clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{
+				Cluster: &clusterv1.Cluster{
+					ObjectMeta: metav1.ObjectMeta{Name: "test-cluster"},
+				},
+				AWSCluster: awsCluster,
+				Client:     client,
+			})
+			if err != nil {
+				t.Fatalf("Failed to create test context: %v", err)
+				return
+			}
+
+			s := NewService(clusterScope)
+
+			id, err := s.getNatGatewayForSubnet(&testCases[idx].input)
+
+			if tc.expectErr && err == nil {
+				t.Fatal("expected error but got no error")
+			}
+			if err != nil && len(tc.expectErrMessage) > 0 {
+				if err.Error() != tc.expectErrMessage {
+					t.Fatalf("got an unexpected error message:\nwant: %v\n got: %v\n", tc.expectErrMessage, err.Error())
+				}
+			}
+			if !tc.expectErr && err != nil {
+				t.Fatalf("got an unexpected error: %v", err)
+			}
+			if len(tc.expect) > 0 {
+				g.Expect(id).To(Equal(tc.expect))
+			}
+		})
+	}
+}

pkg/cloud/services/network/routetables.go

@@ -369,10 +369,13 @@ func (s *Service) getRouteTableTagParams(id string, public bool, zone string) in
 func (s *Service) getRoutesToPublicSubnet(sn *infrav1.SubnetSpec) ([]*ec2.CreateRouteInput, error) {
 	var routes []*ec2.CreateRouteInput
 
-	if s.scope.VPC().InternetGatewayID == nil {
-		return routes, errors.Errorf("failed to create routing tables: internet gateway for %q is nil", s.scope.VPC().ID)
+	if sn.IsEdge() && sn.IsIPv6 {
+		return nil, errors.Errorf("can't determine routes for unsupported ipv6 subnet in zone type %q", sn.ZoneType)
 	}
 
+	if s.scope.VPC().InternetGatewayID == nil {
+		return routes, errors.Errorf("failed to create routing tables: internet gateway for VPC %q is not present", s.scope.VPC().ID)
+	}
 	routes = append(routes, s.getGatewayPublicRoute())
 	if sn.IsIPv6 {
 		routes = append(routes, s.getGatewayPublicIPv6Route())
@@ -382,7 +385,13 @@ func (s *Service) getRoutesToPublicSubnet(sn *infrav1.SubnetSpec) ([]*ec2.Create
 }
 
 func (s *Service) getRoutesToPrivateSubnet(sn *infrav1.SubnetSpec) (routes []*ec2.CreateRouteInput, err error) {
-	natGatewayID, err := s.getNatGatewayForSubnet(sn)
+	var natGatewayID string
+
+	if sn.IsEdge() && sn.IsIPv6 {
+		return nil, errors.Errorf("can't determine routes for unsupported ipv6 subnet in zone type %q", sn.ZoneType)
+	}
+
+	natGatewayID, err = s.getNatGatewayForSubnet(sn)
 	if err != nil {
 		return routes, err
 	}