feat(natgw): make bgp speaker configured by CRDs

Signed-off-by: SkalaNetworks <[email protected]>

charts/kube-ovn/templates/kube-ovn-crd.yaml

@@ -508,7 +508,23 @@ spec:
                   properties:
                     enabled:
                       type: boolean
-                    parameters:
+                    asn:
+                      type: integer
+                    remoteAsn:
+                      type: integer
+                    neighbors:
+                      type: array
+                      items:
+                        type: string
+                    holdTime:
+                      type: string
+                    routerId:
+                      type: string
+                    password:
+                      type: string
+                    enableGracefulRestart:
+                      type: boolean
+                    extraArgs:
                       type: array
                       items:
                         type: string

dist/images/install.sh

@@ -745,7 +745,23 @@ spec:
                   properties:
                     enabled:
                       type: boolean
-                    parameters:
+                    asn:
+                      type: integer
+                    remoteAsn:
+                      type: integer
+                    neighbors:
+                      type: array
+                      items:
+                        type: string
+                    holdTime:
+                      type: string
+                    routerId:
+                      type: string
+                    password:
+                      type: string
+                    enableGracefulRestart:
+                      type: boolean
+                    extraArgs:
                       type: array
                       items:
                         type: string

pkg/apis/kubeovn/v1/types.go

@@ -528,8 +528,15 @@ type VpcNatSpec struct {
 }
 
 type VpcBgpSpeaker struct {
-	Enabled    bool     `json:"enabled"`
-	Parameters []string `json:"parameters"`
+	Enabled               bool            `json:"enabled"`
+	ASN                   uint32          `json:"asn"`
+	RemoteASN             uint32          `json:"remoteAsn"`
+	Neighbors             []string        `json:"neighbors"`
+	HoldTime              metav1.Duration `json:"holdTime"`
+	RouterID              string          `json:"routerId"`
+	Password              string          `json:"password"`
+	EnableGracefulRestart bool            `json:"enableGracefulRestart"`
+	ExtraArgs             []string        `json:"extraArgs"`
 }
 
 type VpcNatStatus struct {

pkg/controller/vpc_nat_gateway.go

@@ -915,11 +915,68 @@ func (c *Controller) genNatGwStatefulSet(gw *kubeovnv1.VpcNatGateway, oldSts *v1
 		}
 
 		args := []string{
-			"--nat-gw-mode",
-			"-v5",
+			"--nat-gw-mode", // Force to run in  NAT GW mode, we're not announcing Pod IPs or Services, only EIPs
 		}
 
-		args = append(args, gw.Spec.BgpSpeaker.Parameters...)
+		speakerParams := gw.Spec.BgpSpeaker
+
+		if speakerParams.RouterID != "" { // Override default auto-selected RouterID
+			args = append(args, fmt.Sprintf("--router-id=%s", speakerParams.RouterID))
+		}
+
+		if speakerParams.Password != "" { // Password for TCP MD5 BGP
+			args = append(args, fmt.Sprintf("--auth-password=%s", speakerParams.Password))
+		}
+
+		if speakerParams.EnableGracefulRestart { // Enable graceful restart
+			args = append(args, "--graceful-restart")
+		}
+
+		if speakerParams.HoldTime != (metav1.Duration{}) { // Hold time
+			args = append(args, fmt.Sprintf("--holdtime=%s", speakerParams.HoldTime.Duration.String()))
+		}
+
+		if speakerParams.ASN == 0 { // The ASN we use to speak
+			return nil, fmt.Errorf("ASN not set, but must be non-zero value")
+		}
+
+		if speakerParams.RemoteASN == 0 { // The ASN we speak to
+			return nil, fmt.Errorf("remote ASN not set, but must be non-zero value")
+		}
+
+		args = append(args, fmt.Sprintf("--cluster-as=%d", speakerParams.ASN))
+		args = append(args, fmt.Sprintf("--neighbor-as=%d", speakerParams.RemoteASN))
+
+		if len(speakerParams.Neighbors) == 0 {
+			return nil, fmt.Errorf("no BGP neighbors specified")
+		}
+
+		var neighIPv4 []string
+		var neighIPv6 []string
+		for _, neighbor := range speakerParams.Neighbors {
+			switch util.CheckProtocol(neighbor) {
+			case kubeovnv1.ProtocolIPv4:
+				neighIPv4 = append(neighIPv4, neighbor)
+			case kubeovnv1.ProtocolIPv6:
+				neighIPv6 = append(neighIPv6, neighbor)
+			}
+		}
+
+		argNeighIPv4 := strings.Join(neighIPv4, ",")
+		argNeighIPv6 := strings.Join(neighIPv6, ",")
+		argNeighIPv4 = fmt.Sprintf("--neighbor-address=%s", argNeighIPv4)
+		argNeighIPv6 = fmt.Sprintf("--neighbor-ipv6-address=%s", argNeighIPv6)
+
+		if len(neighIPv4) > 0 {
+			args = append(args, argNeighIPv4)
+		}
+
+		if len(neighIPv6) > 0 {
+			args = append(args, argNeighIPv6)
+		}
+
+		// Extra args to start the speaker with, for example, logging levels...
+		args = append(args, speakerParams.ExtraArgs...)
 
 		sts.Spec.Template.Spec.ServiceAccountName = "vpc-nat-gw"
 		speakerContainer := corev1.Container{

pkg/speaker/bgp.go

@@ -19,29 +19,29 @@ import (
 
 var maskMap = map[string]int{kubeovnv1.ProtocolIPv4: 32, kubeovnv1.ProtocolIPv6: 128}
 
-// reconciliateRoutes configures the BGP speaker to announce only the routes we are expected to announce
+// reconcileRoutes configures the BGP speaker to announce only the routes we are expected to announce
 // and to withdraw the ones that should not be announced anymore
-func (c *Controller) reconciliateRoutes(expectedPrefixes prefixMap) error {
+func (c *Controller) reconcileRoutes(expectedPrefixes prefixMap) error {
 	if len(c.config.NeighborAddresses) != 0 {
-		err := c.reconciliateIPFamily(kubeovnv1.ProtocolIPv4, expectedPrefixes)
+		err := c.reconcileIPFamily(kubeovnv1.ProtocolIPv4, expectedPrefixes)
 		if err != nil {
-			return fmt.Errorf("failed to reconciliate IPv4 routes: %w", err)
+			return fmt.Errorf("failed to reconcile IPv4 routes: %w", err)
 		}
 	}
 
 	if len(c.config.NeighborIPv6Addresses) != 0 {
-		err := c.reconciliateIPFamily(kubeovnv1.ProtocolIPv6, expectedPrefixes)
+		err := c.reconcileIPFamily(kubeovnv1.ProtocolIPv6, expectedPrefixes)
 		if err != nil {
-			return fmt.Errorf("failed to reconciliate IPv6 routes: %w", err)
+			return fmt.Errorf("failed to reconcile IPv6 routes: %w", err)
 		}
 	}
 
 	return nil
 }
 
-// reconciliateIPFamily announces prefixes we are not currently announcing and withdraws prefixes we should
+// reconcileIPFamily announces prefixes we are not currently announcing and withdraws prefixes we should
 // not be announcing for a given IP family (IPv4/IPv6)
-func (c *Controller) reconciliateIPFamily(ipFamily string, expectedPrefixes prefixMap) error {
+func (c *Controller) reconcileIPFamily(ipFamily string, expectedPrefixes prefixMap) error {
 	// Get the address family associated with the Kube-OVN family
 	afi, err := kubeOvnFamilyToAFI(ipFamily)
 	if err != nil {

pkg/speaker/config.go

@@ -73,7 +73,7 @@ func ParseFlags() (*Configuration, error) {
 		argAnnounceClusterIP           = pflag.BoolP("announce-cluster-ip", "", false, "The Cluster IP of the service to announce to the BGP peers.")
 		argGrpcHost                    = pflag.String("grpc-host", "127.0.0.1", "The host address for grpc to listen, default: 127.0.0.1")
 		argGrpcPort                    = pflag.Uint32("grpc-port", DefaultBGPGrpcPort, "The port for grpc to listen, default:50051")
-		argClusterAs                   = pflag.Uint32("cluster-as", DefaultBGPClusterAs, "The as number of container network, default 65000")
+		argClusterAs                   = pflag.Uint32("cluster-as", DefaultBGPClusterAs, "The AS number of container network, default 65000")
 		argRouterID                    = pflag.String("router-id", "", "The address for the speaker to use as router id, default the node ip")
 		argNodeIPs                     = pflag.String("node-ips", "", "The comma-separated list of node IP addresses to use instead of the pod IP address for the next hop router IP address.")
 		argNeighborAddress             = pflag.String("neighbor-address", "", "Comma separated IPv4 router addresses the speaker connects to.")

pkg/speaker/eip.go

@@ -51,5 +51,5 @@ func (c *Controller) announceEIPs(eips []*v1.IptablesEIP) error {
 		}
 	}
 
-	return c.reconciliateRoutes(expectedPrefixes)
+	return c.reconcileRoutes(expectedPrefixes)
 }

pkg/speaker/subnet.go

@@ -115,7 +115,7 @@ func (c *Controller) syncSubnetRoutes() {
 		}
 	}
 
-	if err := c.reconciliateRoutes(bgpExpected); err != nil {
-		klog.Errorf("failed to reconciliate routes: %s", err.Error())
+	if err := c.reconcileRoutes(bgpExpected); err != nil {
+		klog.Errorf("failed to reconcile routes: %s", err.Error())
 	}
 }