From affc90e1333ee0f86561869775dc272fbb25f04a Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 13 Sep 2021 11:08:20 +0800 Subject: [PATCH] Add global except list for egress to avoid SNAT (#2707) For some environment, some destination(not podCIDR/svcCIDR) can be communicate with each other directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li --- build/yamls/antrea-aks.yml | 12 ++++++++---- build/yamls/antrea-eks.yml | 12 ++++++++---- build/yamls/antrea-gke.yml | 12 ++++++++---- build/yamls/antrea-ipsec.yml | 12 ++++++++---- build/yamls/antrea.yml | 12 ++++++++---- build/yamls/base/conf/antrea-agent.conf | 4 ++++ build/yamls/flow-aggregator.yml | 4 ++-- cmd/antrea-agent/agent.go | 9 +++++++++ cmd/antrea-agent/config.go | 6 ++++++ cmd/antrea-agent/options.go | 8 ++++++++ pkg/agent/agent.go | 5 ++++- pkg/agent/config/node_config.go | 6 ++++++ pkg/agent/openflow/client.go | 17 +++++++++++++---- pkg/agent/openflow/pipeline.go | 16 +++++++++++++--- pkg/agent/openflow/testing/mock_openflow.go | 8 ++++---- test/integration/agent/openflow_test.go | 3 ++- 16 files changed, 111 insertions(+), 35 deletions(-) diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index e66e7d206ec..761ad1d8143 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3929,6 +3929,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4131,7 +4135,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k namespace: kube-system --- apiVersion: v1 @@ -4202,7 +4206,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-4d7ch86gch + value: antrea-config-2f9d9ch78k image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4253,7 +4257,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k name: antrea-config - name: antrea-controller-tls secret: @@ -4534,7 +4538,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 507171c6a40..27abc3fe35e 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3929,6 +3929,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4131,7 +4135,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k namespace: kube-system --- apiVersion: v1 @@ -4202,7 +4206,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-4d7ch86gch + value: antrea-config-2f9d9ch78k image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4253,7 +4257,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k name: antrea-config - name: antrea-controller-tls secret: @@ -4536,7 +4540,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-4d7ch86gch + name: antrea-config-2f9d9ch78k name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index edcd10cc3f0..708b564d6a1 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3929,6 +3929,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4131,7 +4135,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-ct7fm8k579 + name: antrea-config-tg4dt2f5c5 namespace: kube-system --- apiVersion: v1 @@ -4202,7 +4206,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-ct7fm8k579 + value: antrea-config-tg4dt2f5c5 image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4253,7 +4257,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-ct7fm8k579 + name: antrea-config-tg4dt2f5c5 name: antrea-config - name: antrea-controller-tls secret: @@ -4537,7 +4541,7 @@ spec: path: /home/kubernetes/bin name: host-cni-bin - configMap: - name: antrea-config-ct7fm8k579 + name: antrea-config-tg4dt2f5c5 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 9db6f081ab8..ac56799d770 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3929,6 +3929,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. @@ -4136,7 +4140,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-7tm5f22tt7 + name: antrea-config-47gkd69mkk namespace: kube-system --- apiVersion: v1 @@ -4216,7 +4220,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-7tm5f22tt7 + value: antrea-config-47gkd69mkk image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4267,7 +4271,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-7tm5f22tt7 + name: antrea-config-47gkd69mkk name: antrea-config - name: antrea-controller-tls secret: @@ -4583,7 +4587,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-7tm5f22tt7 + name: antrea-config-47gkd69mkk name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 554a1277d7a..32310929fa3 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3929,6 +3929,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. @@ -4136,7 +4140,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-4g55dbc872 + name: antrea-config-6chb77thbh namespace: kube-system --- apiVersion: v1 @@ -4207,7 +4211,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-4g55dbc872 + value: antrea-config-6chb77thbh image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4258,7 +4262,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-4g55dbc872 + name: antrea-config-6chb77thbh name: antrea-config - name: antrea-controller-tls secret: @@ -4539,7 +4543,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-4g55dbc872 + name: antrea-config-6chb77thbh name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 0dcd5b4519e..52ae4e97183 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -93,6 +93,10 @@ wireGuard: # The port for WireGuard to receive traffic. # port: 51820 +egress: +# exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. +# exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. diff --git a/build/yamls/flow-aggregator.yml b/build/yamls/flow-aggregator.yml index ccde1187516..4cc47d24aed 100644 --- a/build/yamls/flow-aggregator.yml +++ b/build/yamls/flow-aggregator.yml @@ -180,7 +180,7 @@ metadata: annotations: {} labels: app: flow-aggregator - name: flow-aggregator-configmap-2k727bgdf4 + name: flow-aggregator-configmap-5df2dmbm8h namespace: flow-aggregator --- apiVersion: v1 @@ -248,7 +248,7 @@ spec: serviceAccountName: flow-aggregator volumes: - configMap: - name: flow-aggregator-configmap-2k727bgdf4 + name: flow-aggregator-configmap-5df2dmbm8h name: flow-aggregator-config - hostPath: path: /var/log/antrea/flow-aggregator diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 25f9da0fecf..ed9389faf12 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -136,6 +136,14 @@ func run(o *Options) error { wireguardConfig := &config.WireGuardConfig{ Port: o.config.WireGuard.Port, } + exceptCIDRs := []net.IPNet{} + for _, cidr := range o.config.Egress.ExceptCIDRs { + _, exceptCIDR, _ := net.ParseCIDR(cidr) + exceptCIDRs = append(exceptCIDRs, *exceptCIDR) + } + egressConfig := &config.EgressConfig{ + ExceptCIDRs: exceptCIDRs, + } routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll) if err != nil { return fmt.Errorf("error creating route client: %v", err) @@ -175,6 +183,7 @@ func run(o *Options) error { serviceCIDRNetv6, networkConfig, wireguardConfig, + egressConfig, networkReadyCh, stopCh, features.DefaultFeatureGate.Enabled(features.AntreaProxy), diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 2ab42d9c14b..f11637d3504 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -176,6 +176,8 @@ type AgentConfig struct { TransportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"` // AntreaProxy contains AntreaProxy related configuration options. AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"` + // Egress related configurations. + Egress EgressConfig `yaml:"egress"` } type AntreaProxyConfig struct { @@ -196,3 +198,7 @@ type WireGuardConfig struct { // The port for the WireGuard to receive traffic. Defaults to 51820. Port int `yaml:"port,omitempty"` } + +type EgressConfig struct { + ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"` +} diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 716497e6029..0c4469e79f2 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -156,6 +156,14 @@ func (o *Options) validate(args []string) error { if err := o.validateFlowExporterConfig(); err != nil { return fmt.Errorf("failed to validate flow exporter config: %v", err) } + if features.DefaultFeatureGate.Enabled(features.Egress) { + for _, cidr := range o.config.Egress.ExceptCIDRs { + _, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("Egress Except CIDR %s is invalid", cidr) + } + } + } return nil } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 23a4aee9650..0f1ee69bf80 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -88,6 +88,7 @@ type Initializer struct { networkConfig *config.NetworkConfig nodeConfig *config.NodeConfig wireGuardConfig *config.WireGuardConfig + egressConfig *config.EgressConfig enableProxy bool // networkReadyCh should be closed once the Node's network is ready. // The CNI server will wait for it before handling any CNI Add requests. @@ -111,6 +112,7 @@ func NewInitializer( serviceCIDRv6 *net.IPNet, networkConfig *config.NetworkConfig, wireGuardConfig *config.WireGuardConfig, + egressConfig *config.EgressConfig, networkReadyCh chan<- struct{}, stopCh <-chan struct{}, enableProxy bool, @@ -132,6 +134,7 @@ func NewInitializer( serviceCIDRv6: serviceCIDRv6, networkConfig: networkConfig, wireGuardConfig: wireGuardConfig, + egressConfig: egressConfig, networkReadyCh: networkReadyCh, stopCh: stopCh, enableProxy: enableProxy, @@ -385,7 +388,7 @@ func (i *Initializer) initOpenFlowPipeline() error { // Install OpenFlow entries to enable Pod traffic to external IP // addresses. - if err := i.ofClient.InstallExternalFlows(); err != nil { + if err := i.ofClient.InstallExternalFlows(i.egressConfig.ExceptCIDRs); err != nil { klog.Errorf("Failed to install openflow entries for external connectivity: %v", err) return err } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index b8ab85d65e3..15c8c081301 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -95,6 +95,10 @@ type WireGuardConfig struct { MTU int } +type EgressConfig struct { + ExceptCIDRs []net.IPNet +} + // Local Node configurations retrieved from K8s API or host networking state. type NodeConfig struct { // The Node's name used in Kubernetes. @@ -130,6 +134,8 @@ type NodeConfig struct { UplinkNetConfig *AdapterNetConfig // The config of the WireGuard interface. WireGuardConfig *WireGuardConfig + // The config of the Egress interface. + EgressConfig *EgressConfig } func (n *NodeConfig) String() string { diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 838b02368a0..d04586c2f8b 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -155,7 +155,7 @@ type Client interface { // Pods to the external IP address, and mark the packets to be SNAT'd // with the configured SNAT IPs. On Windows Node, the flows also perform // SNAT with the Openflow NAT action. - InstallExternalFlows() error + InstallExternalFlows(exceptCIDRs []net.IPNet) error // InstallSNATMarkFlows installs flows for a local SNAT IP. On Linux, a // single flow is added to mark the packets tunnelled from remote Nodes @@ -796,15 +796,24 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo return connCh, c.initialize() } -func (c *client) InstallExternalFlows() error { +func (c *client) InstallExternalFlows(exceptCIDRs []net.IPNet) error { localGatewayMAC := c.nodeConfig.GatewayConfig.MAC var flows []binding.Flow + var ipv4CIDRs []net.IPNet + var ipv6CIDRs []net.IPNet + for _, cidr := range exceptCIDRs { + if cidr.IP.To4() == nil { + ipv6CIDRs = append(ipv6CIDRs, cidr) + } else { + ipv4CIDRs = append(ipv4CIDRs, cidr) + } + } if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs) } if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...) } if err := c.ofEntryOperations.AddAll(flows); err != nil { return fmt.Errorf("failed to install flows for external communication: %v", err) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 6d3d4db2ede..288023380d3 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -313,6 +313,7 @@ type client struct { replayMutex sync.RWMutex nodeConfig *config.NodeConfig networkConfig *config.NetworkConfig + egressConfig *config.EgressConfig gatewayOFPort uint32 // ovsDatapathType is the type of the datapath used by the bridge. ovsDatapathType ovsconfig.OVSDatapathType @@ -1990,7 +1991,7 @@ func (c *client) snatSkipNodeFlow(nodeIP net.IP, category cookie.Category) bindi // snatCommonFlows installs the default flows for performing SNAT for traffic to // the external network. The flows identify the packets to external, and send // them to SNATTable, where SNAT IPs are looked up for the packets. -func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { +func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet, category cookie.Category) []binding.Flow { nextTable := L3ForwardingTable.GetNext() ipProto := getIPProtocol(localSubnet.IP) flows := []binding.Flow{ @@ -2042,6 +2043,15 @@ func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGate Cookie(c.cookieAllocator.Request(category).Raw()). Done(), } + for _, cidr := range exceptCIDRs { + flows = append(flows, L3ForwardingTable.BuildFlow(priorityNormal). + MatchProtocol(ipProto). + MatchRegMark(FromLocalRegMark). + MatchDstIPNet(cidr). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } return flows } @@ -2376,11 +2386,11 @@ func (c *client) decTTLFlows(category cookie.Category) []binding.Flow { } // externalFlows returns the flows needed to enable SNAT for external traffic. -func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow { +func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet) []binding.Flow { if !c.enableEgress { return nil } - return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT) + return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, exceptCIDRs, cookie.SNAT) } // policyConjKeyFuncKeyFunc knows how to get key of a *policyRuleConjunction. diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 5f2636e7251..9e1d6219d4e 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -337,17 +337,17 @@ func (mr *MockClientMockRecorder) InstallEndpointFlows(arg0, arg1 interface{}) * } // InstallExternalFlows mocks base method -func (m *MockClient) InstallExternalFlows() error { +func (m *MockClient) InstallExternalFlows(arg0 []net.IPNet) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallExternalFlows") + ret := m.ctrl.Call(m, "InstallExternalFlows", arg0) ret0, _ := ret[0].(error) return ret0 } // InstallExternalFlows indicates an expected call of InstallExternalFlows -func (mr *MockClientMockRecorder) InstallExternalFlows() *gomock.Call { +func (mr *MockClientMockRecorder) InstallExternalFlows(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallExternalFlows", reflect.TypeOf((*MockClient)(nil).InstallExternalFlows)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallExternalFlows", reflect.TypeOf((*MockClient)(nil).InstallExternalFlows), arg0) } // InstallGatewayFlows mocks base method diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 0392b5e8f1f..5f0e6eb9616 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -242,7 +242,8 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { } func testExternalFlows(t *testing.T, config *testConfig) { - if err := c.InstallExternalFlows(); err != nil { + exceptCIDRs := []net.IPNet{} + if err := c.InstallExternalFlows(exceptCIDRs); err != nil { t.Errorf("Failed to install OpenFlow entries to allow Pod to communicate to the external addresses: %v", err) }