Add global except list for egress to avoid SNAT (antrea-io#2707)

For some environment, some destination(not podCIDR/svcCIDR) can be communicate with each other directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li [email protected]
leonstack · Sep 10, 2021 · 27b8428 · 27b8428
1 parent dded211
commit 27b8428
Show file tree

Hide file tree

Showing 14 changed files with 87 additions and 12 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -93,6 +93,10 @@ wireGuard:
 #  The port for WireGuard to receive traffic.
 #  port: 51820
 
+egressExcept:
+#  The cidrs will ignore SNAT action when Egress enabled
+#  cidrs: []
+
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
 # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -129,6 +129,14 @@ func run(o *Options) error {
 	wireguardConfig := &config.WireGuardConfig{
 		Port: o.config.WireGuard.Port,
 	}
+	excidrs := []net.IPNet{}
+	for _, cidr := range o.config.EgressExcept.CIDRs {
+		_, excidr, _ := net.ParseCIDR(cidr)
+		excidrs = append(excidrs, *excidr)
+	}
+	egressExceptConfig := &config.EgressExceptConfig{
+		CIDRs: excidrs,
+	}
 	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
@@ -158,6 +166,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		wireguardConfig,
+		egressExceptConfig,
 		networkReadyCh,
 		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -162,9 +162,15 @@ type AgentConfig struct {
 	// If there are multiple IP addresses configured on the interface, the first one is used.
 	// The interface configured with Node IP is used if this parameter is not set.
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// Egress related configurations.
+	EgressExcept EgressExceptConfig `yaml:"egressExcept"`
 }
 
 type WireGuardConfig struct {
 	// The port for the WireGuard to receive traffic. Defaults to 51820.
 	Port int `yaml:"port,omitempty"`
 }
+
+type EgressExceptConfig struct {
+	CIDRs []string `yaml:"cidrs,omitempty"`
+}
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -153,6 +153,14 @@ func (o *Options) validate(args []string) error {
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
+	if features.DefaultFeatureGate.Enabled(features.Egress) {
+		for _, cidr := range o.config.EgressExcept.CIDRs {
+			_, _, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("Egress Except CIDR %s is invalid", cidr)
+			}
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -83,6 +83,7 @@ type Initializer struct {
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	wireGuardConfig *config.WireGuardConfig
+	egressExceptConfig *config.EgressExceptConfig
 	enableProxy     bool
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
@@ -103,6 +104,7 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	wireGuardConfig *config.WireGuardConfig,
+	egressExceptConfig *config.EgressExceptConfig,
 	networkReadyCh chan<- struct{},
 	stopCh <-chan struct{},
 	enableProxy bool) *Initializer {
@@ -119,6 +121,7 @@ func NewInitializer(
 		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		wireGuardConfig: wireGuardConfig,
+		egressExceptConfig: egressExceptConfig,
 		networkReadyCh:  networkReadyCh,
 		stopCh:          stopCh,
 		enableProxy:     enableProxy,
@@ -355,7 +358,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 	roundInfo := getRoundInfo(i.ovsBridgeClient)
 
 	// Set up all basic flows.
-	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig)
+	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig, i.egressExceptConfig)
 	if err != nil {
 		klog.Errorf("Failed to initialize openflow client: %v", err)
 		return err

diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go
@@ -80,6 +80,10 @@ type WireGuardConfig struct {
 	MTU int
 }
 
+type EgressExceptConfig struct {
+	CIDRs []net.IPNet
+}
+
 // Local Node configurations retrieved from K8s API or host networking state.
 type NodeConfig struct {
 	// The Node's name used in Kubernetes.
@@ -115,6 +119,7 @@ type NodeConfig struct {
 	UplinkNetConfig *AdapterNetConfig
 	// The config of the WireGuard interface.
 	WireGuardConfig *WireGuardConfig
+	EgressExceptConfig *EgressExceptConfig
 }
 
 func (n *NodeConfig) String() string {

diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go
@@ -40,7 +40,7 @@ type Client interface {
 	// be called to ensure that the set of OVS flows is correct. All flows programmed in the
 	// switch which match the current round number will be deleted before any new flow is
 	// installed.
-	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig) (<-chan struct{}, error)
+	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error)
 
 	// InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist.
 	InstallGatewayFlows() error
@@ -729,9 +729,10 @@ func (c *client) initialize() error {
 	return nil
 }
 
-func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig) (<-chan struct{}, error) {
+func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error) {
 	c.nodeConfig = nodeConfig
 	c.networkConfig = networkConfig
+	c.egressExceptConfig = egressExceptConfig
 
 	if config.IsIPv4Enabled(nodeConfig, c.networkConfig.TrafficEncapMode) {
 		c.ipProtocols = append(c.ipProtocols, binding.ProtocolIP)
@@ -778,11 +779,20 @@ func (c *client) InstallExternalFlows() error {
 	localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
 
 	var flows []binding.Flow
+	var ipv4CIDRs []net.IPNet
+	var ipv6CIDRs []net.IPNet
+	for _, cidr := range c.egressExceptConfig.CIDRs{
+		if cidr.IP.To4() == nil {
+			ipv6CIDRs = append(ipv6CIDRs, cidr)
+		} else {
+			ipv4CIDRs = append(ipv4CIDRs, cidr)
+		}
+	}
 	if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil {
-		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
+		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs)
 	}
 	if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil {
-		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...)
 	}
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return fmt.Errorf("failed to install flows for external communication: %v", err)

diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go
@@ -322,6 +322,7 @@ type client struct {
 	replayMutex   sync.RWMutex
 	nodeConfig    *config.NodeConfig
 	networkConfig *config.NetworkConfig
+	egressExceptConfig *config.EgressExceptConfig
 	gatewayOFPort uint32
 	// ovsDatapathType is the type of the datapath used by the bridge.
 	ovsDatapathType ovsconfig.OVSDatapathType
@@ -1857,7 +1858,7 @@ func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Catego
 // snatCommonFlows installs the default flows for performing SNAT for traffic to
 // the external network. The flows identify the packets to external, and send
 // them to snatTable, where SNAT IPs are looked up for the packets.
-func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow {
+func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet, category cookie.Category) []binding.Flow {
 	l3FwdTable := c.pipeline[l3ForwardingTable]
 	nextTable := l3FwdTable.GetNext()
 	ipProto := getIPProtocol(localSubnet.IP)
@@ -1910,6 +1911,15 @@ func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGate
 			Cookie(c.cookieAllocator.Request(category).Raw()).
 			Done(),
 	}
+	for _, cidr := range exceptCIDRs {
+		flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).
+			MatchProtocol(ipProto).
+			MatchRegMark(FromLocalRegMark).
+			MatchDstIPNet(cidr).
+			Action().GotoTable(nextTable).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done(),)
+	}
 	return flows
 }
 
@@ -2174,11 +2184,11 @@ func (c *client) decTTLFlows(category cookie.Category) []binding.Flow {
 }
 
 // externalFlows returns the flows needed to enable SNAT for external traffic.
-func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow {
+func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, cidrs []net.IPNet) []binding.Flow {
 	if !c.enableEgress {
 		return nil
 	}
-	return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT)
+	return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cidrs, cookie.SNAT)
 }
 
 // policyConjKeyFuncKeyFunc knows how to get key of a *policyRuleConjunction.

diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go