diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index f69807a46c4..201ecc4958d 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -79,6 +79,14 @@ spec: oneOf: - format: ipv4 - format: ipv6 + excepts: + items: + properties: + cidr: + type: string + format: cidr + type: object + type: array externalIPPool: type: string status: diff --git a/pkg/agent/controller/egress/egress_controller.go b/pkg/agent/controller/egress/egress_controller.go index 3061caeb413..2cf9aa6f75e 100644 --- a/pkg/agent/controller/egress/egress_controller.go +++ b/pkg/agent/controller/egress/egress_controller.go @@ -19,6 +19,7 @@ import ( "fmt" "net" "reflect" + "sort" "strings" "sync" "time" @@ -81,6 +82,8 @@ type egressState struct { egressIP string // The actual datapath mark of this Egress. Used to check if the mark changes since last process. mark uint32 + // The except Info for bypass SNAT flows + excepts []crdv1a2.Except // The actual openflow ports for which we have installed SNAT rules. Used to identify stale openflow ports when // updating or deleting an Egress. ofPorts sets.Int32 @@ -464,11 +467,12 @@ func (c *EgressController) deleteEgressState(egressName string) { delete(c.egressStates, egressName) } -func (c *EgressController) newEgressState(egressName string, egressIP string) *egressState { +func (c *EgressController) newEgressState(egressName string, egressIP string, excepts []crdv1a2.Except) *egressState { c.egressStatesMutex.Lock() defer c.egressStatesMutex.Unlock() state := &egressState{ egressIP: egressIP, + excepts: excepts, ofPorts: sets.NewInt32(), pods: sets.NewString(), } @@ -592,7 +596,7 @@ func (c *EgressController) syncEgress(egressName string) error { return nil } if !exist { - eState = c.newEgressState(egressName, egress.Spec.EgressIP) + eState = c.newEgressState(egressName, egress.Spec.EgressIP, egress.Spec.Excepts) } localNodeSelected, err := c.cluster.ShouldSelectEgress(egress) @@ -619,12 +623,13 @@ func (c *EgressController) syncEgress(egressName string) error { // If the mark changes, uninstall all of the Egress's Pod flows first, then installs them with new mark. // It could happen when the Egress IP is added to or removed from the Node. - if eState.mark != mark { + if eState.mark != mark || !Equal(eState.excepts, egress.Spec.Excepts) { // Uninstall all of its Pod flows. if err := c.uninstallPodFlows(egressName, eState, eState.ofPorts, eState.pods); err != nil { return err } eState.mark = mark + eState.excepts = egress.Spec.Excepts } if err := c.updateEgressStatus(egress, c.localIPDetector.IsLocalIP(egress.Spec.EgressIP)); err != nil { @@ -671,7 +676,7 @@ func (c *EgressController) syncEgress(egressName string) error { staleOFPorts.Delete(ofPort) continue } - if err := c.ofClient.InstallPodSNATFlows(uint32(ofPort), egressIP, mark); err != nil { + if err := c.ofClient.InstallPodSNATFlows(uint32(ofPort), egressIP, mark, eState.excepts); err != nil { return err } eState.ofPorts.Insert(ofPort) @@ -872,3 +877,17 @@ func (c *EgressController) deleteEgressGroup(group *cpv1b2.EgressGroup) { delete(c.egressGroups, group.Name) c.queue.Add(group.Name) } + +func Equal(a, b []crdv1a2.Except) bool { + if len(a) != len(b) { + return false + } + sort.Slice(a, func(i, j int) bool {return a[i].CIDR < a[j].CIDR}) + sort.Slice(b, func(i, j int) bool {return b[i].CIDR < b[j].CIDR}) + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/pkg/agent/controller/egress/egress_controller_test.go b/pkg/agent/controller/egress/egress_controller_test.go index 31e54b7cf97..62b8af325cb 100644 --- a/pkg/agent/controller/egress/egress_controller_test.go +++ b/pkg/agent/controller/egress/egress_controller_test.go @@ -50,6 +50,7 @@ const ( fakeLocalEgressIP1 = "1.1.1.1" fakeLocalEgressIP2 = "1.1.1.2" fakeRemoteEgressIP1 = "1.1.1.3" + fakeEgressExceptCIDR1 = "1.1.1.0/24" fakeNode = "node1" ) @@ -79,6 +80,13 @@ type antreaClientGetter struct { clientset versioned.Interface } +func getExcepts() []crdv1a2.Except { + var exceptCIDR crdv1a2.Except + exceptCIDR = crdv1a2.Except{CIDR: fakeEgressExceptCIDR1,} + var excepts []crdv1a2.Except + excepts = append(excepts, exceptCIDR) + return excepts +} func (g *antreaClientGetter) GetAntreaClient() (versioned.Interface, error) { return g.clientset, nil } @@ -144,6 +152,7 @@ func newFakeController(t *testing.T, initObjects []runtime.Object) *fakeControll } func TestSyncEgress(t *testing.T) { + tests := []struct { name string existingEgress *crdv1a2.Egress @@ -158,11 +167,11 @@ func TestSyncEgress(t *testing.T) { name: "Local IP becomes non local", existingEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, existingEgressGroup: &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -182,13 +191,13 @@ func TestSyncEgress(t *testing.T) { expectedEgresses: []*crdv1a2.Egress{ { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) @@ -198,8 +207,8 @@ func TestSyncEgress(t *testing.T) { mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(2)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(0)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(0)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(0), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(0), getExcepts()) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) }, }, @@ -236,8 +245,8 @@ func TestSyncEgress(t *testing.T) { }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1)) @@ -245,8 +254,8 @@ func TestSyncEgress(t *testing.T) { mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeRemoteEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeRemoteEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) }, @@ -255,11 +264,11 @@ func TestSyncEgress(t *testing.T) { name: "Change from local Egress IP to another local one", existingEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2, Excepts: getExcepts()}, }, existingEgressGroup: &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -278,14 +287,14 @@ func TestSyncEgress(t *testing.T) { expectedEgresses: []*crdv1a2.Egress{ { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) @@ -297,8 +306,8 @@ func TestSyncEgress(t *testing.T) { mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2) mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP2), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP2), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP2), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP2), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2) }, @@ -307,7 +316,7 @@ func TestSyncEgress(t *testing.T) { name: "Change from local Egress IP to a remote one", existingEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -335,8 +344,8 @@ func TestSyncEgress(t *testing.T) { }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) @@ -347,8 +356,8 @@ func TestSyncEgress(t *testing.T) { mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) }, }, @@ -360,7 +369,7 @@ func TestSyncEgress(t *testing.T) { }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, existingEgressGroup: &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -379,13 +388,13 @@ func TestSyncEgress(t *testing.T) { expectedEgresses: []*crdv1a2.Egress{ { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) mockOFClient.EXPECT().UninstallPodSNATFlows(uint32(1)) @@ -394,8 +403,8 @@ func TestSyncEgress(t *testing.T) { mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) }, @@ -404,11 +413,11 @@ func TestSyncEgress(t *testing.T) { name: "Add an Egress having overlapping Pods", existingEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2, Excepts: getExcepts()}, }, existingEgressGroup: &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -427,24 +436,24 @@ func TestSyncEgress(t *testing.T) { expectedEgresses: []*crdv1a2.Egress{ { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP2, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP2), uint32(2)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(2)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP2), uint32(2), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP2), uint32(2)) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP2) @@ -455,11 +464,11 @@ func TestSyncEgress(t *testing.T) { name: "Add an Egress sharing the same Egress IP and having overlapping Pods", existingEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, newEgress: &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, }, existingEgressGroup: &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -478,21 +487,21 @@ func TestSyncEgress(t *testing.T) { expectedEgresses: []*crdv1a2.Egress{ { ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, { ObjectMeta: metav1.ObjectMeta{Name: "egressB", UID: "uidB"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, Status: crdv1a2.EgressStatus{EgressNode: fakeNode}, }, }, expectedCalls: func(mockOFClient *openflowtest.MockClient, mockRouteClient *routetest.MockInterface, mockIPAssigner *ipassignertest.MockIPAssigner) { mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1).Times(3) }, }, @@ -543,7 +552,7 @@ func TestSyncEgress(t *testing.T) { func TestSyncOverlappingEgress(t *testing.T) { egress1 := &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, } egressGroup1 := &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA"}, @@ -567,7 +576,7 @@ func TestSyncOverlappingEgress(t *testing.T) { // egress3 shares a Pod with egress1 and has the same EgressIP. egress3 := &crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressC", UID: "uidC"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, } egressGroup3 := &cpv1b2.EgressGroup{ ObjectMeta: metav1.ObjectMeta{Name: "egressC", UID: "uidC"}, @@ -595,21 +604,21 @@ func TestSyncOverlappingEgress(t *testing.T) { c.queue.Done(item) c.mockOFClient.EXPECT().InstallSNATMarkFlows(net.ParseIP(fakeLocalEgressIP1), uint32(1)) - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1)) - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) c.mockRouteClient.EXPECT().AddSNATRule(net.ParseIP(fakeLocalEgressIP1), uint32(1)) c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) err := c.syncEgress(egress1.Name) assert.NoError(t, err) // egress2's IP is not local and pod1 has enforced egress1, so only one Pod SNAT flow is expected. - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(3), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) c.mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) err = c.syncEgress(egress2.Name) assert.NoError(t, err) // egress3 shares the same IP as egress1 and pod2 has enforced egress1, so only one Pod SNAT flow is expected. - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(4), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(4), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) err = c.syncEgress(egress3.Name) assert.NoError(t, err) @@ -638,13 +647,13 @@ func TestSyncOverlappingEgress(t *testing.T) { assert.ElementsMatch(t, []string{egress2.Name, egress3.Name}, pendingItems) // pod1 is expected to enforce egress2. - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0)) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(1), net.ParseIP(fakeRemoteEgressIP1), uint32(0), getExcepts()) c.mockIPAssigner.EXPECT().UnassignIP(fakeRemoteEgressIP1) err = c.syncEgress(egress2.Name) assert.NoError(t, err) // pod2 is expected to enforce egress3. - c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1)) + c.mockOFClient.EXPECT().InstallPodSNATFlows(uint32(2), net.ParseIP(fakeLocalEgressIP1), uint32(1), getExcepts()) c.mockIPAssigner.EXPECT().UnassignIP(fakeLocalEgressIP1) err = c.syncEgress(egress3.Name) assert.NoError(t, err) @@ -745,7 +754,7 @@ func TestUpdateEgressStatus(t *testing.T) { t.Run(tt.name, func(t *testing.T) { egress := crdv1a2.Egress{ ObjectMeta: metav1.ObjectMeta{Name: "egressA", UID: "uidA", ResourceVersion: "fake-ResourceVersion"}, - Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1}, + Spec: crdv1a2.EgressSpec{EgressIP: fakeLocalEgressIP1, Excepts: getExcepts()}, } fakeClient := &fakeversioned.Clientset{} getCalled := 0 diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 5803ba56813..29a2385c9c4 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -26,6 +26,7 @@ import ( "antrea.io/antrea/pkg/agent/openflow/cookie" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" + crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" binding "antrea.io/antrea/pkg/ovs/openflow" utilip "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/third_party/proxy" @@ -176,7 +177,7 @@ type Client interface { // tunnel destination, and the packets should be SNAT'd on the remote // Node. As of now, a Pod can be configured to use only a single SNAT // IP in a single address family (IPv4 or IPv6). - InstallPodSNATFlows(ofPort uint32, snatIP net.IP, snatMark uint32) error + InstallPodSNATFlows(ofPort uint32, snatIP net.IP, snatMark uint32, excepts []crdv1a2.Except) error // UninstallPodSNATFlows removes the SNAT flows for the local Pod. UninstallPodSNATFlows(ofPort uint32) error @@ -806,8 +807,8 @@ func (c *client) UninstallSNATMarkFlows(mark uint32) error { return c.deleteFlows(c.snatFlowCache, cacheKey) } -func (c *client) InstallPodSNATFlows(ofPort uint32, snatIP net.IP, snatMark uint32) error { - flows := []binding.Flow{c.snatRuleFlow(ofPort, snatIP, snatMark, c.nodeConfig.GatewayConfig.MAC)} +func (c *client) InstallPodSNATFlows(ofPort uint32, snatIP net.IP, snatMark uint32, excepts []crdv1a2.Except) error { + flows := c.snatRuleFlows(ofPort, snatIP, snatMark, c.nodeConfig.GatewayConfig.MAC, excepts) cacheKey := fmt.Sprintf("p%x", ofPort) c.replayMutex.RLock() defer c.replayMutex.RUnlock() diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 7adf2cc0a77..6d1ca34ac3c 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -33,6 +33,7 @@ import ( "antrea.io/antrea/pkg/agent/metrics" "antrea.io/antrea/pkg/agent/openflow/cookie" "antrea.io/antrea/pkg/agent/types" + crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" binding "antrea.io/antrea/pkg/ovs/openflow" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/ovs/ovsctl" @@ -1927,26 +1928,39 @@ func (c *client) snatIPFromTunnelFlow(snatIP net.IP, mark uint32) binding.Flow { Done() } -// snatRuleFlow generates a flow that applies the SNAT rule for a local Pod. If +// snatRuleFlows generates flows that applies the SNAT rule for a local Pod. If // the SNAT IP exists on the local Node, it sets the packet mark with the ID of // the SNAT IP, for the traffic from the ofPort to external; if the SNAT IP is // on a remote Node, it tunnels the packets to the SNAT IP. -func (c *client) snatRuleFlow(ofPort uint32, snatIP net.IP, snatMark uint32, localGatewayMAC net.HardwareAddr) binding.Flow { +func (c *client) snatRuleFlows(ofPort uint32, snatIP net.IP, snatMark uint32, localGatewayMAC net.HardwareAddr, excepts []crdv1a2.Except) []binding.Flow { ipProto := getIPProtocol(snatIP) snatTable := c.pipeline[snatTable] + l3FwdTable := c.pipeline[l3ForwardingTable] + nextTable := l3FwdTable.GetNext() + snatFlows := []binding.Flow{} if snatMark != 0 { // Local SNAT IP. - return snatTable.BuildFlow(priorityNormal). + return append(snatFlows, snatTable.BuildFlow(priorityNormal). MatchProtocol(ipProto). MatchCTStateNew(true).MatchCTStateTrk(true). MatchInPort(ofPort). Action().LoadPktMarkRange(snatMark, snatPktMarkRange). Action().GotoTable(snatTable.GetNext()). Cookie(c.cookieAllocator.Request(cookie.SNAT).Raw()). - Done() + Done()) + } + for _, except := range excepts { + _, excidr, _ := net.ParseCIDR(except.CIDR) + snatFlows = append(snatFlows, l3FwdTable.BuildFlow(priorityNormal). + MatchInPort(ofPort). + MatchProtocol(ipProto). + MatchDstIPNet(*excidr). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(cookie.SNAT).Raw()). + Done()) } // SNAT IP should be on a remote Node. - return snatTable.BuildFlow(priorityNormal). + snatFlows = append(snatFlows, snatTable.BuildFlow(priorityNormal). MatchProtocol(ipProto). MatchInPort(ofPort). Action().SetSrcMAC(localGatewayMAC). @@ -1955,7 +1969,8 @@ func (c *client) snatRuleFlow(ofPort uint32, snatIP net.IP, snatMark uint32, loc Action().SetTunnelDst(snatIP). Action().GotoTable(l3DecTTLTable). Cookie(c.cookieAllocator.Request(cookie.SNAT).Raw()). - Done() + Done()) + return snatFlows } // loadBalancerServiceFromOutsideFlow generates the flow to forward LoadBalancer service traffic from outside node diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index abf0bfe9f2e..c78ca7e5c4f 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -26,6 +26,7 @@ import ( ip "antrea.io/antrea/pkg/util/ip" proxy "antrea.io/antrea/third_party/proxy" gomock "github.com/golang/mock/gomock" + crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" net "net" reflect "reflect" ) @@ -406,17 +407,17 @@ func (mr *MockClientMockRecorder) InstallPodFlows(arg0, arg1, arg2, arg3 interfa } // InstallPodSNATFlows mocks base method -func (m *MockClient) InstallPodSNATFlows(arg0 uint32, arg1 net.IP, arg2 uint32) error { +func (m *MockClient) InstallPodSNATFlows(arg0 uint32, arg1 net.IP, arg2 uint32, arg3 []crdv1a2.Except) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallPodSNATFlows", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "InstallPodSNATFlows", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // InstallPodSNATFlows indicates an expected call of InstallPodSNATFlows -func (mr *MockClientMockRecorder) InstallPodSNATFlows(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) InstallPodSNATFlows(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPodSNATFlows", reflect.TypeOf((*MockClient)(nil).InstallPodSNATFlows), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPodSNATFlows", reflect.TypeOf((*MockClient)(nil).InstallPodSNATFlows), arg0, arg1, arg2, arg3) } // InstallPolicyRuleFlows mocks base method diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 437c3058e01..0cd5091d525 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -212,6 +212,10 @@ type EgressStatus struct { EgressNode string `json:"egressNode"` } +type Except struct { + CIDR string `json:"cidr"` +} + // EgressSpec defines the desired state for Egress. type EgressSpec struct { // AppliedTo selects Pods to which the Egress will be applied. @@ -226,6 +230,7 @@ type EgressSpec struct { // If it is non-empty, the EgressIP will be assigned to a Node specified by the pool automatically and will failover // to a different Node when the Node becomes unreachable. ExternalIPPool string `json:"externalIPPool"` + Excepts []Except `json:"excepts"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object