Create network resources for remote AZs

TBD

api/bases/octavia.openstack.org_octavias.yaml

@@ -106,6 +106,13 @@ spec:
                 description: OctaviaLbMgmtNetworks Settings for Octavia management
                   networks
                 properties:
+                  availabilityZoneCIDRs:
+                    additionalProperties:
+                      type: string
+                    description: 'AvailabilityZoneCIDRs are the CIDRs of each management
+                      network associated with an Availability Zone (ex: {"az":"172.34.0.0/24",
+                      ...})'
+                    type: object
                   availabilityZones:
                     description: Availability zones for the octavia management network
                       resources

api/v1beta1/octavia_types.go

@@ -229,6 +229,10 @@ type OctaviaLbMgmtNetworks struct {
 	// +kubebuilder:validation:Optional
 	// Availability zones for the octavia management network resources
 	AvailabilityZones []string `json:"availabilityZones,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// AvailabilityZoneCIDRs are the CIDRs of each management network associated with an Availability Zone (ex: {"az":"172.34.0.0/24", ...})
+	AvailabilityZoneCIDRs map[string]string `json:"availabilityZoneCIDRs,omitempty"`
 }
 
 // OctaviaAmphoraFlavor Settings for custom Amphora flavors

config/crd/bases/octavia.openstack.org_octavias.yaml

@@ -106,6 +106,13 @@ spec:
                 description: OctaviaLbMgmtNetworks Settings for Octavia management
                   networks
                 properties:
+                  availabilityZoneCIDRs:
+                    additionalProperties:
+                      type: string
+                    description: 'AvailabilityZoneCIDRs are the CIDRs of each management
+                      network associated with an Availability Zone (ex: {"az":"172.34.0.0/24",
+                      ...})'
+                    type: object
                   availabilityZones:
                     description: Availability zones for the octavia management network
                       resources

controllers/amphoracontroller_controller.go

@@ -600,8 +600,8 @@ func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps(
 			rsyslogIPAddresses = append(rsyslogIPAddresses, fmt.Sprintf("%s:514", val))
 		}
 	}
-	ipAddresses = sort.Strings(ipAddresses)
-	rsyslogIPAddresses = sort.Strings(rsyslogIPAddresses)
+	sort.Strings(ipAddresses)
+	sort.Strings(rsyslogIPAddresses)
 	ipAddressString := strings.Join(ipAddresses, ",")
 	templateParameters["ControllerIPList"] = ipAddressString
 	templateParameters["AdminLogTargetList"] = strings.Join(rsyslogIPAddresses, ",")

pkg/octavia/lb_mgmt_network.go

@@ -17,6 +17,7 @@ package octavia
 import (
 	"context"
 	"fmt"
+	"net/netip"
 
 	"github.com/go-logr/logr"
 	"github.com/gophercloud/gophercloud"
@@ -74,31 +75,38 @@ func findPort(client *gophercloud.ServiceClient, networkID string, name string,
 	return nil, nil
 }
 
-func ensurePort(client *gophercloud.ServiceClient, tenantNetwork *networks.Network, securityGroups *[]string, log *logr.Logger) (*ports.Port, error) {
-	p, err := findPort(client, tenantNetwork.ID, LbMgmtRouterPortName, log)
+func ensurePort(client *gophercloud.ServiceClient, availabilityZone *string, tenantNetwork *networks.Network, securityGroups *[]string, log *logr.Logger) (*ports.Port, bool, error) {
+	var portName string
+	if availabilityZone == nil {
+		portName = LbMgmtRouterPortName
+	} else {
+		portName = fmt.Sprintf(LbMgmtRouterPortNameAZ, *availabilityZone)
+	}
+
+	p, err := findPort(client, tenantNetwork.ID, portName, log)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if p != nil {
 		//
 		// TODO(beagles): reconcile port properties? Is there anything to do? Security groups possibly.
 		//
-		return p, nil
+		return p, false, nil
 	}
 	log.Info("Unable to locate port, creating new one")
 	asu := true
 	createOpts := ports.CreateOpts{
-		Name:           LbMgmtRouterPortName,
+		Name:           portName,
 		AdminStateUp:   &asu,
 		NetworkID:      tenantNetwork.ID,
 		SecurityGroups: securityGroups,
 	}
 	p, err = ports.Create(client, createOpts).Extract()
 	if err != nil {
 		log.Error(err, "Error creating port")
-		return nil, err
+		return nil, false, err
 	}
-	return p, nil
+	return p, true, nil
 }
 
 func ensureSubnet(client *gophercloud.ServiceClient, ipVersion int, createOpts subnets.CreateOpts, log *logr.Logger) (*subnets.Subnet, error) {
@@ -342,7 +350,8 @@ func ensureLbMgmtSubnetRoutes(
 	networkParameters *NetworkParameters,
 	tenantRouterPort *ports.Port,
 ) error {
-	if len(tenantSubnet.HostRoutes) == 0 {
+	if len(tenantSubnet.HostRoutes) == 0 ||
+		tenantSubnet.HostRoutes[0].NextHop != tenantRouterPort.FixedIPs[0].IPAddress {
 		hostRoutes := []subnets.HostRoute{
 			{
 				DestinationCIDR: networkParameters.ProviderCIDR.String(),
@@ -363,11 +372,23 @@ func ensureLbMgmtSubnetRoutes(
 
 func ensureLbMgmtSubnet(
 	client *gophercloud.ServiceClient,
+	availabilityZone *string,
 	tenantNetwork *networks.Network,
 	networkParameters *NetworkParameters,
 	log *logr.Logger,
 ) (*subnets.Subnet, error) {
 	var ipVersion int
+
+	var subnetName string
+	var description string
+	if availabilityZone == nil {
+		subnetName = LbMgmtSubnetName
+		description = LbMgmtSubnetDescription
+	} else {
+		subnetName = fmt.Sprintf(LbMgmtSubnetNameAZ, *availabilityZone)
+		description = fmt.Sprintf(LbMgmtSubnetDescriptionAZ, *availabilityZone)
+	}
+
 	if networkParameters.TenantCIDR.Addr().Is6() {
 		ipVersion = 6
 	} else {
@@ -378,8 +399,8 @@ func ensureLbMgmtSubnet(
 	if ipVersion == 6 {
 		gatewayIP := LbMgmtSubnetIPv6GatewayIP
 		createOpts = subnets.CreateOpts{
-			Name:            LbMgmtSubnetName,
-			Description:     LbMgmtSubnetDescription,
+			Name:            subnetName,
+			Description:     description,
 			NetworkID:       tenantNetwork.ID,
 			TenantID:        tenantNetwork.TenantID,
 			CIDR:            networkParameters.TenantCIDR.String(),
@@ -397,8 +418,8 @@ func ensureLbMgmtSubnet(
 	} else {
 		gatewayIP := LbMgmtSubnetGatewayIP
 		createOpts = subnets.CreateOpts{
-			Name:        LbMgmtSubnetName,
-			Description: LbMgmtSubnetDescription,
+			Name:        subnetName,
+			Description: description,
 			NetworkID:   tenantNetwork.ID,
 			TenantID:    tenantNetwork.TenantID,
 			CIDR:        networkParameters.TenantCIDR.String(),
@@ -415,28 +436,41 @@ func ensureLbMgmtSubnet(
 	return ensureSubnet(client, ipVersion, createOpts, log)
 }
 
-func getLbMgmtNetwork(client *gophercloud.ServiceClient, serviceTenantID string) (*networks.Network, error) {
-	return getNetwork(client, LbMgmtNetName, serviceTenantID)
-}
-
-func ensureLbMgmtNetwork(client *gophercloud.ServiceClient, networkDetails *octaviav1.OctaviaLbMgmtNetworks,
-	serviceTenantID string, log *logr.Logger) (*networks.Network, error) {
-	mgmtNetwork, err := getLbMgmtNetwork(client, serviceTenantID)
+func ensureLbMgmtNetwork(
+	client *gophercloud.ServiceClient,
+	availabilityZone *string,
+	networkDetails *octaviav1.OctaviaLbMgmtNetworks,
+	serviceTenantID string,
+	log *logr.Logger,
+) (*networks.Network, error) {
+	var networkName string
+	var description string
+	var azHints []string
+	if availabilityZone == nil {
+		networkName = LbMgmtNetName
+		description = LbMgmtNetDescription
+		azHints = networkDetails.AvailabilityZones
+	} else {
+		networkName = fmt.Sprintf(LbMgmtNetNameAZ, *availabilityZone)
+		description = fmt.Sprintf(LbMgmtNetDescriptionAZ, *availabilityZone)
+		azHints = []string{*availabilityZone}
+	}
+	mgmtNetwork, err := getNetwork(client, networkName, serviceTenantID)
 	if err != nil {
 		return nil, err
 	}
 
 	if networkDetails == nil && mgmtNetwork == nil {
-		return nil, fmt.Errorf("Cannot find network \"%s\"", LbMgmtNetName)
+		return nil, fmt.Errorf("Cannot find network \"%s\"", networkName)
 	}
 
 	asu := true
 	createOpts := networks.CreateOpts{
-		Name:                  LbMgmtNetName,
-		Description:           LbMgmtNetDescription,
+		Name:                  networkName,
+		Description:           description,
 		AdminStateUp:          &asu,
 		TenantID:              serviceTenantID,
-		AvailabilityZoneHints: networkDetails.AvailabilityZones,
+		AvailabilityZoneHints: azHints,
 	}
 	mgmtNetwork, err = ensureNetwork(client, createOpts, log, serviceTenantID)
 	if err != nil {
@@ -794,11 +828,11 @@ func EnsureAmphoraManagementNetwork(
 		return NetworkProvisioningSummary{}, err
 	}
 
-	tenantNetwork, err := ensureLbMgmtNetwork(client, netDetails, serviceTenant.ID, log)
+	tenantNetwork, err := ensureLbMgmtNetwork(client, nil, netDetails, serviceTenant.ID, log)
 	if err != nil {
 		return NetworkProvisioningSummary{}, err
 	}
-	tenantSubnet, err := ensureLbMgmtSubnet(client, tenantNetwork, networkParameters, log)
+	tenantSubnet, err := ensureLbMgmtSubnet(client, nil, tenantNetwork, networkParameters, log)
 	if err != nil {
 		return NetworkProvisioningSummary{}, err
 	}
@@ -814,7 +848,7 @@ func EnsureAmphoraManagementNetwork(
 
 	securityGroups := []string{lbMgmtSecurityGroupID, lbHealthSecurityGroupID}
 
-	tenantRouterPort, err := ensurePort(client, tenantNetwork, &securityGroups, log)
+	tenantRouterPort, _, err := ensurePort(client, nil, tenantNetwork, &securityGroups, log)
 	if err != nil {
 		return NetworkProvisioningSummary{}, err
 	}
@@ -889,6 +923,53 @@ func EnsureAmphoraManagementNetwork(
 		log.Error(err, fmt.Sprintf("Unable to set host routes on subnet %s", tenantSubnet.ID))
 	}
 
+	for az, cidr := range netDetails.AvailabilityZoneCIDRs {
+		// Create Management network and subnet for AZ
+		network, err := ensureLbMgmtNetwork(client, &az, netDetails, adminTenant.ID, log)
+		if err != nil {
+			return NetworkProvisioningSummary{}, err
+		}
+
+		subnetCIDR, err := netip.ParsePrefix(cidr)
+		if err != nil {
+			return NetworkProvisioningSummary{}, fmt.Errorf("cannot parse CIDR %s for AZ %s: %w", cidr, az, err)
+		}
+		start, end := GetRangeFromCIDR(subnetCIDR)
+		networkAZParameters := NetworkParameters{
+			TenantCIDR:            subnetCIDR,
+			TenantAllocationStart: start,
+			TenantAllocationEnd:   end,
+		}
+		subnet, err := ensureLbMgmtSubnet(client, &az, network, &networkAZParameters, log)
+		if err != nil {
+			return NetworkProvisioningSummary{}, err
+		}
+
+		// Create a port for the router, will be the gateway from the subnet to the control plane
+		routerPort, created, err := ensurePort(client, &az, network, &securityGroups, log)
+		if err != nil {
+			return NetworkProvisioningSummary{}, err
+		}
+
+		if created {
+			// Plug port into the existing router
+			interfaceOpts := routers.AddInterfaceOpts{
+				PortID: routerPort.ID,
+			}
+
+			_, err = routers.AddInterface(client, router.ID, interfaceOpts).Extract()
+			if err != nil {
+				log.Error(err, fmt.Sprintf("Unable to add interface port %s to router %s", tenantRouterPort.ID, router.ID))
+			}
+		}
+
+		// Set route to the control plane
+		err = ensureLbMgmtSubnetRoutes(client, subnet, networkParameters, routerPort)
+		if err != nil {
+			log.Error(err, fmt.Sprintf("Unable to set host routes on subnet %s", tenantSubnet.ID))
+		}
+	}
+
 	return NetworkProvisioningSummary{
 		TenantNetworkID:         tenantNetwork.ID,
 		TenantSubnetID:          tenantSubnet.ID,

pkg/octavia/network_consts.go

@@ -24,15 +24,27 @@ const (
 	// LbMgmtNetName -
 	LbMgmtNetName = "lb-mgmt-net"
 
+	// LbMgmtNetNameAZ -
+	LbMgmtNetNameAZ = "lb-mgmt-%s-net"
+
 	// LbMgmtNetDescription -
 	LbMgmtNetDescription = "LBaaS Management Network"
 
+	// LbMgmtNetDescriptionAZ -
+	LbMgmtNetDescriptionAZ = "LBaaS Management Network for %s"
+
 	// LbMgmtSubnetName -
 	LbMgmtSubnetName = "lb-mgmt-subnet"
 
+	// LbMgmtSubnetNameAZ -
+	LbMgmtSubnetNameAZ = "lb-mgmt-%s-subnet"
+
 	// LbMgmtSubnetDescription -
 	LbMgmtSubnetDescription = "LBaaS Management Subnet"
 
+	// LbMgmtSubnetDescriptionAZ -
+	LbMgmtSubnetDescriptionAZ = "LBaaS Management Subnet for %s"
+
 	// IPv4 consts
 
 	// LbMgmtSubnetGatewayIP -
@@ -79,6 +91,9 @@ const (
 	// LbMgmtRouterPortName
 	LbMgmtRouterPortName = "lb-mgmt-router-port"
 
+	// LbMgmtRouterPortNameAZ
+	LbMgmtRouterPortNameAZ = "lb-mgmt-%s-router-port"
+
 	// Network attachment details
 	// LbNetworkAttachmentName
 	LbNetworkAttachmentName = "octavia"

pkg/octavia/network_parameters.go

@@ -50,8 +50,8 @@ func getConfigFromNAD(
 	return nadConfig, nil
 }
 
-// getRangeFromCIDR - compute a IP address range from a CIDR
-func getRangeFromCIDR(
+// GetRangeFromCIDR - compute a IP address range from a CIDR
+func GetRangeFromCIDR(
 	cidr netip.Prefix,
 ) (start netip.Addr, end netip.Addr) {
 	// For IPv6, a /64 is expected, if the CIDR is aaaa:bbbb:cccc:dddd::/64,
@@ -140,7 +140,7 @@ func GetNetworkParametersFromNAD(
 	}
 
 	// Compute an allocation range based on the CIDR
-	start, end := getRangeFromCIDR(networkParameters.TenantCIDR)
+	start, end := GetRangeFromCIDR(networkParameters.TenantCIDR)
 	networkParameters.TenantAllocationStart = start
 	networkParameters.TenantAllocationEnd = end