Implement the controller for API BGPPolicy

This commit implements the controller of API `BGPPolicy`, designed to advertise
Service IPs, Egress IPs, and Pod IPs to BGP peers from selected Kubernetes Nodes.

According to the spec of `BGPPolicy`, the Node selector is used to select Nodes
to which a `BGPPolicy` is applied. Multiple `BGPPolicies` can be applied to the
same Node. However, only the oldest `BGPPolicy` will be effective on a Node,
with others serving as alternatives. The effective one may be changed in the
following cases:

- The current effective BGPPolicy is updated and not applied to the Node.
- The current effective BGPPolicy is deleted.

The BGP server instance is only created and started for the effective BGPPolicy on
a Node. If the effective BGPPolicy is changed, the corresponding BGP server instance
will be terminated by calling the `Stop` method, and a new BGP server instance will
be created and started by calling the `Start` method for the new effective BGPPolicy.

To create a BGP server instance, ASN, router ID, and listen port must be specified.
The ASN and listen port are specified in the spec of the effective BGPPolicy. For router ID,
if the Kubernetes cluster is IPv4-only or dual-stack, we use the Node's IPv4 address
as the router ID, ensuring uniqueness. If the Kubernetes cluster is IPv6-only, where no
Node IPv4 address is available, the router ID could be specified via the Node annotation
`node.antrea.io/bgp-router-id`. If not present, a router ID will be generated by hashing
the Node name and update it to the Node annotation `node.antrea.io/bgp-router-id`.
Additionally, the stale BGP server instance will be terminated and a new BGP server
instance should be created and started when any of ASN, routerID, or listen port changes.

The information of the BGP peers is specified in the effective BGPPolicy. The unique
identification of a BGP peer is the peer IP address and peer ASN.

To reconcile the latest BGP peers:

- Get the BGP peers to be added and add them by calling the `AddPeer` method of the
  BGP server instance.
- Get the BGP peers to be deleted and delete them by calling the `RemovePeer` method
  of the BGP server instance.
- Get the remaining BGP peers and calculate the updated BGP peers, then update them by
  calling the `UpdatePeer` method of the BGP server instance.

The information of the IPs to be advertised can be calculated from the spec of the
effective BGPPolicy. Currently, we advertise the IPs and CIDRs to all the BGP peers.

To reconcile the latest IPs to all BGP peers:

- If the BGP server instance is newly created and started, advertise all the IPs by
  calling the `AdvertiseRoutes` method.
- If the BGP server instance is not newly created and started:
  - Get the IPs/CIDRs to be added and advertise them by calling the `AdvertiseRoutes` method.
  - Get the IPs/CIDRs to be removed and withdraw them by calling the `WithdrawRoutes` method.

The feature is gated by the alpha `BGPPolicy` feature gate and only supported in Linux.

Signed-off-by: Hongliang Liu <[email protected]>

pkg/agent/controller/bgp/controller.go

@@ -110,7 +110,8 @@ type Controller struct {
 	endpointSliceLister       discoverylisters.EndpointSliceLister
 	endpointSliceListerSynced cache.InformerSynced
 
-	secretInformer cache.SharedIndexInformer
+	secretInformer     cache.SharedIndexInformer
+	secretListerSynced cache.InformerSynced
 
 	bgpPolicyState *bgpPolicyState
 
@@ -226,6 +227,7 @@ func NewBGPPolicyController(nodeInformer coreinformers.NodeInformer,
 		UpdateFunc: c.updateSecret,
 		DeleteFunc: c.deleteSecret,
 	})
+	c.secretListerSynced = c.secretInformer.HasSynced
 
 	return c, nil
 }
@@ -241,6 +243,7 @@ func (c *Controller) Run(ctx context.Context) {
 		c.serviceListerSynced,
 		c.bgpPolicyListerSynced,
 		c.endpointSliceListerSynced,
+		c.serviceListerSynced,
 	}
 	if c.egressEnabled {
 		cacheSyncs = append(cacheSyncs, c.egressListerSynced)
@@ -250,6 +253,7 @@ func (c *Controller) Run(ctx context.Context) {
 	}
 
 	go wait.UntilWithContext(ctx, c.worker, time.Second)
+	go c.secretInformer.Run(ctx.Done())
 
 	<-ctx.Done()
 }
@@ -896,17 +900,20 @@ func (c *Controller) updateNode(oldObj, obj interface{}) {
 
 func (c *Controller) addSecret(obj interface{}) {
 	secret := obj.(*corev1.Secret)
+	klog.V(2).InfoS("Processing Secret ADD event", "Secret", klog.KObj(secret))
 	c.updateBGPPeerPasswords(secret)
 	c.queue.Add(dummyKey)
 }
 
 func (c *Controller) updateSecret(_, obj interface{}) {
 	secret := obj.(*corev1.Secret)
+	klog.V(2).InfoS("Processing Secret UPDATE event", "Secret", klog.KObj(secret))
 	c.updateBGPPeerPasswords(secret)
 	c.queue.Add(dummyKey)
 }
 
-func (c *Controller) deleteSecret(_ interface{}) {
+func (c *Controller) deleteSecret(obj interface{}) {
+	klog.V(2).InfoS("Processing Secret DELETE event", "Secret", klog.KObj(obj.(*corev1.Secret)))
 	c.updateBGPPeerPasswords(nil)
 	c.queue.Add(dummyKey)
 }

test/e2e/bgppolicy_test.go

@@ -24,10 +24,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/utils/ptr"
 
@@ -75,6 +75,7 @@ func routesToStrings(routes []FRRRoute) []string {
 
 func TestBGPPolicy(t *testing.T) {
 	skipIfNotIPv4Cluster(t)
+	skipIfHasWindowsNodes(t)
 	skipIfExternalFRRNotSet(t)
 
 	data, err := setupTest(t)
@@ -217,10 +218,13 @@ func checkFRRRouterBGPRoutes(t *testing.T, expectedRouteStrings, notExpectedRout
 			return false, err
 		}
 		gotRouteStrings := routesToStrings(gotRoutes)
-		if assert.NotSubset(t, gotRouteStrings, expectedRouteStrings) {
+		gotRoutesSet := sets.NewString(gotRouteStrings...)
+		notExpectedRoutesSet := sets.NewString(notExpectedRouteStrings...)
+		expectedRoutesSet := sets.NewString(expectedRouteStrings...)
+		if !gotRoutesSet.IsSuperset(expectedRoutesSet) {
 			return false, nil
 		}
-		if len(notExpectedRouteStrings) != 0 && assert.Subset(t, gotRouteStrings, notExpectedRouteStrings) {
+		if notExpectedRoutesSet.Len() != 0 && gotRoutesSet.IsSuperset(notExpectedRoutesSet) {
 			return false, nil
 		}
 		return true, nil