From e635aef7e3b2afccdf7d6d2ba3b56840723e5942 Mon Sep 17 00:00:00 2001 From: Quan Tian Date: Thu, 4 May 2023 17:27:46 +0800 Subject: [PATCH] Update Egress API to support multiple Egress IPs and pools (#4603) Nodes in a cluster could reside in multiple subnets, but the egress IP needs to be routable in the underlay network and usually resides in the same subnet of the Node hosting it. Therefore, there may be a situation where no available nodes are eligible to host an egress IP address if all Nodes in a subnet are down, interrupting egress traffic of the workloads that use the IP. As the first step of supporting the above scenario, the patch extends the Egress IP to support multiple Egress IPs and pools so that one Egress IP can be configured for each subnet, making Egress failover across the whole cluster possible. Besides, it also adds a field, `status.egressIP`, to represent the effective Egress IP. When there is no eligible Node for any of the Egress IPs, the field will be empty. Signed-off-by: Quan Tian --- build/charts/antrea/crds/egress.yaml | 34 +++++++-- build/yamls/antrea-aks.yml | 34 +++++++-- build/yamls/antrea-crds.yml | 34 +++++++-- build/yamls/antrea-eks.yml | 34 +++++++-- build/yamls/antrea-gke.yml | 34 +++++++-- build/yamls/antrea-ipsec.yml | 34 +++++++-- build/yamls/antrea.yml | 34 +++++++-- .../controller/egress/egress_controller.go | 31 ++++++-- .../egress/egress_controller_test.go | 73 +++++++++++++++---- pkg/apis/crd/v1alpha2/types.go | 12 ++- .../crd/v1alpha2/zz_generated.deepcopy.go | 12 ++- pkg/controller/egress/controller.go | 12 ++- pkg/controller/egress/validate.go | 6 ++ 13 files changed, 309 insertions(+), 75 deletions(-) diff --git a/build/charts/antrea/crds/egress.yaml b/build/charts/antrea/crds/egress.yaml index bcdfea97652..e47ac0fc47c 100644 --- a/build/charts/antrea/crds/egress.yaml +++ b/build/charts/antrea/crds/egress.yaml @@ -20,11 +20,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -82,16 +88,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 2b5cbf397dd..56f13f6eecb 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1083,11 +1083,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1145,16 +1151,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 86f6099c28b..01a41d38565 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -1074,11 +1074,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1136,16 +1142,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 1d7bd09b0a4..97b869e9a03 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1083,11 +1083,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1145,16 +1151,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 332e1bc4969..e3ceaf2ad93 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1083,11 +1083,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1145,16 +1151,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 8f320edc824..cef78e82f20 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1083,11 +1083,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1145,16 +1151,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 5ec4e750882..567204110fd 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1083,11 +1083,17 @@ spec: type: object required: - appliedTo - anyOf: - - required: - - egressIP - - required: - - externalIPPool + oneOf: + - anyOf: + - required: + - egressIP + - required: + - externalIPPool + - anyOf: + - required: + - egressIPs + - required: + - externalIPPools properties: appliedTo: type: object @@ -1145,16 +1151,30 @@ spec: oneOf: - format: ipv4 - format: ipv6 + egressIPs: + type: array + items: + type: string + oneOf: + - maxLength: 0 + - format: ipv4 + - format: ipv6 externalIPPool: type: string + externalIPPools: + type: array + items: + type: string status: type: object properties: egressNode: type: string + egressIP: + type: string additionalPrinterColumns: - - description: Specifies the SNAT IP address for the selected workloads. - jsonPath: .spec.egressIP + - description: The effective SNAT IP address for the selected workloads. + jsonPath: .status.egressIP name: EgressIP type: string - jsonPath: .metadata.creationTimestamp diff --git a/pkg/agent/controller/egress/egress_controller.go b/pkg/agent/controller/egress/egress_controller.go index 65112ee821a..50c1071b6c5 100644 --- a/pkg/agent/controller/egress/egress_controller.go +++ b/pkg/agent/controller/egress/egress_controller.go @@ -198,7 +198,16 @@ func NewEgressController( if !ok { return nil, fmt.Errorf("obj is not Egress: %+v", obj) } - return []string{egress.Spec.EgressIP}, nil + var egressIPs []string + if egress.Spec.EgressIP != "" { + egressIPs = append(egressIPs, egress.Spec.EgressIP) + } + for _, egressIP := range egress.Spec.EgressIPs { + if egressIP != "" { + egressIPs = append(egressIPs, egressIP) + } + } + return egressIPs, nil }, }) c.egressInformer.AddEventHandlerWithResyncPeriod( @@ -327,11 +336,11 @@ func (c *EgressController) replaceEgressIPs() error { desiredLocalEgressIPs := sets.NewString() egresses, _ := c.egressLister.List(labels.Everything()) for _, egress := range egresses { - if isEgressSchedulable(egress) && egress.Status.EgressNode == c.nodeName { - desiredLocalEgressIPs.Insert(egress.Spec.EgressIP) + if isEgressSchedulable(egress) && egress.Status.EgressNode == c.nodeName && egress.Status.EgressIP != "" { + desiredLocalEgressIPs.Insert(egress.Status.EgressIP) // Record the Egress's state as we assign their IPs to this Node in the following call. It makes sure these // Egress IPs will be unassigned when the Egresses are deleted. - c.newEgressState(egress.Name, egress.Spec.EgressIP) + c.newEgressState(egress.Name, egress.Status.EgressIP) } } if err := c.ipAssigner.InitIPs(desiredLocalEgressIPs); err != nil { @@ -552,22 +561,28 @@ func (c *EgressController) unbindPodEgress(pod, egress string) (string, bool) { return "", false } -func (c *EgressController) updateEgressStatus(egress *crdv1a2.Egress, isLocal bool) error { +func (c *EgressController) updateEgressStatus(egress *crdv1a2.Egress, egressIP string) error { + isLocal := false + if egressIP != "" { + isLocal = c.localIPDetector.IsLocalIP(egressIP) + } toUpdate := egress.DeepCopy() var updateErr, getErr error if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { if isLocal { // Do nothing if the current EgressNode in status is already this Node. - if toUpdate.Status.EgressNode == c.nodeName { + if toUpdate.Status.EgressNode == c.nodeName && toUpdate.Status.EgressIP == egressIP { return nil } toUpdate.Status.EgressNode = c.nodeName + toUpdate.Status.EgressIP = egressIP } else { // Do nothing if the current EgressNode in status is not this Node. if toUpdate.Status.EgressNode != c.nodeName { return nil } toUpdate.Status.EgressNode = "" + toUpdate.Status.EgressIP = "" } klog.V(2).InfoS("Updating Egress status", "Egress", egress.Name, "oldNode", egress.Status.EgressNode, "newNode", toUpdate.Status.EgressNode) _, updateErr = c.crdClient.CrdV1alpha2().Egresses().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}) @@ -633,7 +648,7 @@ func (c *EgressController) syncEgress(egressName string) error { } // Do not proceed if EgressIP is empty. if desiredEgressIP == "" { - if err := c.updateEgressStatus(egress, false); err != nil { + if err := c.updateEgressStatus(egress, ""); err != nil { return fmt.Errorf("update Egress %s status error: %v", egressName, err) } return nil @@ -670,7 +685,7 @@ func (c *EgressController) syncEgress(egressName string) error { eState.mark = mark } - if err := c.updateEgressStatus(egress, c.localIPDetector.IsLocalIP(desiredEgressIP)); err != nil { + if err := c.updateEgressStatus(egress, desiredEgressIP); err != nil { return fmt.Errorf("update Egress %s status error: %v", egressName, err) } diff --git a/pkg/agent/controller/egress/egress_controller_test.go b/pkg/agent/controller/egress/egress_controller_test.go index 2bc7e21dca9..7875e3da047 100644 --- a/pkg/agent/controller/egress/egress_controller_test.go +++ b/pkg/agent/controller/egress/egress_controller_test.go @@ -281,7 +281,7 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeRemoteEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeRemoteEgressIP1, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -328,7 +328,7 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP2, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -429,7 +429,7 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -477,12 +477,12 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP2, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -528,12 +528,12 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -574,12 +574,12 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2, ExternalIPPool: "external-ip-pool"}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP2, EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -622,12 +622,12 @@ func TestSyncEgress(t *testing.T) { { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, ExternalIPPool: "external-ip-pool"}, - Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, Spec: crdv1a2.EgressSpec{EgressIP: fakeRemoteEgressIP1, ExternalIPPool: "external-ip-pool"}, - Status: crdv1a2.EgressStatus{EgressNode: ""}, + Status: crdv1a2.EgressStatus{}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { @@ -640,6 +640,52 @@ func TestSyncEgress(t *testing.T) { mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) }, }, + { + name: "Remove Egress IP", + maxEgressIPsPerNode: 1, + existingEgress: &crdv1a2.Egress{ + ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, ExternalIPPool: "external-ip-pool"}, + }, + newEgress: &crdv1a2.Egress{ + ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, + Spec: crdv1a2.EgressSpec{ExternalIPPool: "external-ip-pool"}, + Status: crdv1a2.EgressStatus{EgressIP: fakeLocalEgressIP1, EgressNode: fakeNode}, + }, + existingEgressGroup: &cpv1b2.EgressGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, + GroupMembers: []cpv1b2.GroupMember{ + {Pod: &cpv1b2.PodReference{Name: "pod1", Namespace: "ns1"}}, + {Pod: &cpv1b2.PodReference{Name: "pod2", Namespace: "ns2"}}, + }, + }, + newEgressGroup: &cpv1b2.EgressGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, + GroupMembers: []cpv1b2.GroupMember{ + {Pod: &cpv1b2.PodReference{Name: "pod3", Namespace: "ns3"}}, + }, + }, + expectedEgresses: []*crdv1a2.Egress{ + { + ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, + Spec: crdv1a2.EgressSpec{ExternalIPPool: "external-ip-pool"}, + Status: crdv1a2.EgressStatus{}, + }, + }, + expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { + mockIPAssigner.EXPECT().AssignIP(fakeLocalEgressIP1) + mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) + + mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) + mockOFClient.EXPECT().UninstallSNATMarkFlows(uint32(1)) + mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1)) + mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2)) + mockRouteClient.EXPECT().DeleteSNATRule(uint32(1)) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -958,10 +1004,11 @@ func TestUpdateEgressStatus(t *testing.T) { return true, &egress, nil }) - c := &EgressController{crdClient: fakeClient, nodeName: fakeNode} + localIPDetector := &fakeLocalIPDetector{localIPs: sets.NewString(fakeLocalEgressIP1)} + c := &EgressController{crdClient: fakeClient, nodeName: fakeNode, localIPDetector: localIPDetector} _, err := c.crdClient.CrdV1alpha2().Egresses().Create(context.TODO(), &egress, metav1.CreateOptions{}) assert.NoError(t, err) - err = c.updateEgressStatus(&egress, true) + err = c.updateEgressStatus(&egress, fakeLocalEgressIP1) if err != tt.expectedError { t.Errorf("Update Egress error not match, got: %v, expected: %v", err, tt.expectedError) } diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 39232ea1a81..113f3e984ee 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -202,6 +202,9 @@ type Egress struct { type EgressStatus struct { // The name of the Node that holds the Egress IP. EgressNode string `json:"egressNode"` + // EgressIP indicates the effective Egress IP for the selected workloads. It could be empty if the Egress IP in spec + // is not assigned to any Node. It's also useful when there are more than one Egress IP specified in spec. + EgressIP string `json:"egressIP"` } // EgressSpec defines the desired state for Egress. @@ -213,11 +216,18 @@ type EgressSpec struct { // If ExternalIPPool is non-empty, it can be empty and will be assigned by Antrea automatically. // If both ExternalIPPool and EgressIP are non-empty, the IP must be in the pool. EgressIP string `json:"egressIP,omitempty"` + // EgressIPs specifies multiple SNAT IP addresses for the selected workloads. + // Cannot be set with EgressIP. + EgressIPs []string `json:"egressIPs,omitempty"` // ExternalIPPool specifies the IP Pool that the EgressIP should be allocated from. // If it is empty, the specified EgressIP must be assigned to a Node manually. // If it is non-empty, the EgressIP will be assigned to a Node specified by the pool automatically and will failover // to a different Node when the Node becomes unreachable. - ExternalIPPool string `json:"externalIPPool"` + ExternalIPPool string `json:"externalIPPool,omitempty"` + // ExternalIPPools specifies multiple unique IP Pools that the EgressIPs should be allocated from. Entries with the + // same index in EgressIPs and ExternalIPPools are correlated. + // Cannot be set with ExternalIPPool. + ExternalIPPools []string `json:"externalIPPools,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index 9ee9725dd92..518d4c6e74d 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -1,7 +1,7 @@ //go:build !ignore_autogenerated // +build !ignore_autogenerated -// Copyright 2022 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -218,6 +218,16 @@ func (in *EgressList) DeepCopyObject() runtime.Object { func (in *EgressSpec) DeepCopyInto(out *EgressSpec) { *out = *in in.AppliedTo.DeepCopyInto(&out.AppliedTo) + if in.EgressIPs != nil { + in, out := &in.EgressIPs, &out.EgressIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalIPPools != nil { + in, out := &in.ExternalIPPools, &out.ExternalIPPools + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/egress/controller.go b/pkg/controller/egress/controller.go index d5e7a00f37b..7ee01702480 100644 --- a/pkg/controller/egress/controller.go +++ b/pkg/controller/egress/controller.go @@ -126,10 +126,16 @@ func NewEgressController(crdClient clientset.Interface, if !ok { return nil, fmt.Errorf("obj is not Egress: %+v", obj) } - if egress.Spec.ExternalIPPool == "" { - return nil, nil + var externalIPPools []string + if egress.Spec.ExternalIPPool != "" { + externalIPPools = append(externalIPPools, egress.Spec.ExternalIPPool) } - return []string{egress.Spec.ExternalIPPool}, nil + for _, externalIPPool := range egress.Spec.ExternalIPPools { + if externalIPPool != "" { + externalIPPools = append(externalIPPools, externalIPPool) + } + } + return externalIPPools, nil }}) c.externalIPAllocator.AddEventHandler(func(ipPool string) { c.enqueueEgresses(ipPool) diff --git a/pkg/controller/egress/validate.go b/pkg/controller/egress/validate.go index 5769dd985bc..88d6b212d06 100644 --- a/pkg/controller/egress/validate.go +++ b/pkg/controller/egress/validate.go @@ -47,6 +47,12 @@ func (c *EgressController) ValidateEgress(review *admv1.AdmissionReview) *admv1. } shouldAllow := func(oldEgress, newEgress *crdv1alpha2.Egress) (bool, string) { + if len(newEgress.Spec.EgressIPs) > 0 { + return false, "spec.egressIPs is not supported yet" + } + if len(newEgress.Spec.ExternalIPPools) > 0 { + return false, "spec.externalIPPools is not supported yet" + } // Allow it if EgressIP and ExternalIPPool don't change. if newEgress.Spec.EgressIP == oldEgress.Spec.EgressIP && newEgress.Spec.ExternalIPPool == oldEgress.Spec.ExternalIPPool { return true, ""