From 89926f389b830a989b2946d34641190c23578aa1 Mon Sep 17 00:00:00 2001 From: Hongliang Liu Date: Fri, 19 Jul 2024 04:44:54 +0800 Subject: [PATCH] Implement the controller for API BGPPolicy This commit implements the controller of API `BGPPolicy`, designed to advertise Service IPs, Egress IPs, and Pod IPs to BGP peers from selected Kubernetes Nodes. According to the spec of `BGPPolicy`, the Node selector is used to select Nodes to which a `BGPPolicy` is applied. Multiple `BGPPolicies` can be applied to the same Node. However, only the oldest `BGPPolicy` will be effective on a Node, with others serving as alternatives. The effective one may be changed in the following cases: - The current effective BGPPolicy is updated and not applied to the Node. - The current effective BGPPolicy is deleted. The BGP server instance is only created and started for the effective BGPPolicy on a Node. If the effective BGPPolicy is changed, the corresponding BGP server instance will be terminated by calling the `Stop` method, and a new BGP server instance will be created and started by calling the `Start` method for the new effective BGPPolicy. To create a BGP server instance, ASN, router ID, and listen port must be specified. The ASN and listen port are specified in the spec of the effective BGPPolicy. For router ID, if the Kubernetes cluster is IPv4-only or dual-stack, we use the Node's IPv4 address as the router ID, ensuring uniqueness. If the Kubernetes cluster is IPv6-only, where no Node IPv4 address is available, the router ID could be specified via the Node annotation `node.antrea.io/bgp-router-id`. If not present, a router ID will be generated by hashing the Node name and update it to the Node annotation `node.antrea.io/bgp-router-id`. Additionally, the stale BGP server instance will be terminated and a new BGP server instance should be created and started when any of ASN, routerID, or listen port changes. The information of the BGP peers is specified in the effective BGPPolicy. The unique identification of a BGP peer is the peer IP address and peer ASN. To reconcile the latest BGP peers: - Get the BGP peers to be added and add them by calling the `AddPeer` method of the BGP server instance. - Get the BGP peers to be deleted and delete them by calling the `RemovePeer` method of the BGP server instance. - Get the remaining BGP peers and calculate the updated BGP peers, then update them by calling the `UpdatePeer` method of the BGP server instance. The information of the IPs to be advertised can be calculated from the spec of the effective BGPPolicy. Currently, we advertise the IPs and CIDRs to all the BGP peers. To reconcile the latest IPs to all BGP peers: - If the BGP server instance is newly created and started, advertise all the IPs by calling the `AdvertiseRoutes` method. - If the BGP server instance is not newly created and started: - Get the IPs/CIDRs to be added and advertise them by calling the `AdvertiseRoutes` method. - Get the IPs/CIDRs to be removed and withdraw them by calling the `WithdrawRoutes` method. The feature is gated by the alpha `BGPPolicy` feature gate and only supported in Linux. Signed-off-by: Hongliang Liu --- pkg/agent/controller/bgp/controller.go | 11 +++++++++-- test/e2e/bgppolicy_test.go | 10 +++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkg/agent/controller/bgp/controller.go b/pkg/agent/controller/bgp/controller.go index 4eb2a44fefb..76a53c01e71 100644 --- a/pkg/agent/controller/bgp/controller.go +++ b/pkg/agent/controller/bgp/controller.go @@ -110,7 +110,8 @@ type Controller struct { endpointSliceLister discoverylisters.EndpointSliceLister endpointSliceListerSynced cache.InformerSynced - secretInformer cache.SharedIndexInformer + secretInformer cache.SharedIndexInformer + secretListerSynced cache.InformerSynced bgpPolicyState *bgpPolicyState @@ -226,6 +227,7 @@ func NewBGPPolicyController(nodeInformer coreinformers.NodeInformer, UpdateFunc: c.updateSecret, DeleteFunc: c.deleteSecret, }) + c.secretListerSynced = c.secretInformer.HasSynced return c, nil } @@ -241,6 +243,7 @@ func (c *Controller) Run(ctx context.Context) { c.serviceListerSynced, c.bgpPolicyListerSynced, c.endpointSliceListerSynced, + c.serviceListerSynced, } if c.egressEnabled { cacheSyncs = append(cacheSyncs, c.egressListerSynced) @@ -250,6 +253,7 @@ func (c *Controller) Run(ctx context.Context) { } go wait.UntilWithContext(ctx, c.worker, time.Second) + go c.secretInformer.Run(ctx.Done()) <-ctx.Done() } @@ -896,17 +900,20 @@ func (c *Controller) updateNode(oldObj, obj interface{}) { func (c *Controller) addSecret(obj interface{}) { secret := obj.(*corev1.Secret) + klog.V(2).InfoS("Processing Secret ADD event", "Secret", klog.KObj(secret)) c.updateBGPPeerPasswords(secret) c.queue.Add(dummyKey) } func (c *Controller) updateSecret(_, obj interface{}) { secret := obj.(*corev1.Secret) + klog.V(2).InfoS("Processing Secret UPDATE event", "Secret", klog.KObj(secret)) c.updateBGPPeerPasswords(secret) c.queue.Add(dummyKey) } -func (c *Controller) deleteSecret(_ interface{}) { +func (c *Controller) deleteSecret(obj interface{}) { + klog.V(2).InfoS("Processing Secret DELETE event", "Secret", klog.KObj(obj.(*corev1.Secret))) c.updateBGPPeerPasswords(nil) c.queue.Add(dummyKey) } diff --git a/test/e2e/bgppolicy_test.go b/test/e2e/bgppolicy_test.go index 7e902934173..91eb67df864 100644 --- a/test/e2e/bgppolicy_test.go +++ b/test/e2e/bgppolicy_test.go @@ -24,10 +24,10 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" @@ -75,6 +75,7 @@ func routesToStrings(routes []FRRRoute) []string { func TestBGPPolicy(t *testing.T) { skipIfNotIPv4Cluster(t) + skipIfHasWindowsNodes(t) skipIfExternalFRRNotSet(t) data, err := setupTest(t) @@ -217,10 +218,13 @@ func checkFRRRouterBGPRoutes(t *testing.T, expectedRouteStrings, notExpectedRout return false, err } gotRouteStrings := routesToStrings(gotRoutes) - if assert.NotSubset(t, gotRouteStrings, expectedRouteStrings) { + gotRoutesSet := sets.NewString(gotRouteStrings...) + notExpectedRoutesSet := sets.NewString(notExpectedRouteStrings...) + expectedRoutesSet := sets.NewString(expectedRouteStrings...) + if !gotRoutesSet.IsSuperset(expectedRoutesSet) { return false, nil } - if len(notExpectedRouteStrings) != 0 && assert.Subset(t, gotRouteStrings, notExpectedRouteStrings) { + if notExpectedRoutesSet.Len() != 0 && gotRoutesSet.IsSuperset(notExpectedRoutesSet) { return false, nil } return true, nil