Add global except list for egress to avoid SNAT (antrea-io#2707)

For some environment, some destination(not podCIDR/svcCIDR) can be communicate with each other directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li <[email protected]>
leonstack · Oct 22, 2021 · affc90e · affc90e
1 parent 6ea909a
commit affc90e
Show file tree

Hide file tree

Showing 16 changed files with 111 additions and 35 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3929,6 +3929,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4131,7 +4135,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4d7ch86gch
+  name: antrea-config-2f9d9ch78k
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4202,7 +4206,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4d7ch86gch
+          value: antrea-config-2f9d9ch78k
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4253,7 +4257,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4d7ch86gch
+          name: antrea-config-2f9d9ch78k
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4534,7 +4538,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4d7ch86gch
+          name: antrea-config-2f9d9ch78k
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3929,6 +3929,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4131,7 +4135,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4d7ch86gch
+  name: antrea-config-2f9d9ch78k
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4202,7 +4206,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4d7ch86gch
+          value: antrea-config-2f9d9ch78k
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4253,7 +4257,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4d7ch86gch
+          name: antrea-config-2f9d9ch78k
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4536,7 +4540,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4d7ch86gch
+          name: antrea-config-2f9d9ch78k
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3929,6 +3929,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4131,7 +4135,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-ct7fm8k579
+  name: antrea-config-tg4dt2f5c5
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4202,7 +4206,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-ct7fm8k579
+          value: antrea-config-tg4dt2f5c5
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4253,7 +4257,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-ct7fm8k579
+          name: antrea-config-tg4dt2f5c5
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4537,7 +4541,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-ct7fm8k579
+          name: antrea-config-tg4dt2f5c5
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3929,6 +3929,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4136,7 +4140,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-7tm5f22tt7
+  name: antrea-config-47gkd69mkk
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4216,7 +4220,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-7tm5f22tt7
+          value: antrea-config-47gkd69mkk
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4267,7 +4271,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-7tm5f22tt7
+          name: antrea-config-47gkd69mkk
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4583,7 +4587,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-7tm5f22tt7
+          name: antrea-config-47gkd69mkk
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3929,6 +3929,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4136,7 +4140,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4g55dbc872
+  name: antrea-config-6chb77thbh
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4207,7 +4211,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4g55dbc872
+          value: antrea-config-6chb77thbh
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4258,7 +4262,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4g55dbc872
+          name: antrea-config-6chb77thbh
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4539,7 +4543,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4g55dbc872
+          name: antrea-config-6chb77thbh
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -93,6 +93,10 @@ wireGuard:
 #  The port for WireGuard to receive traffic.
 #  port: 51820
 
+egress:
+#  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+#  exceptCIDRs: []
+
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
 # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/build/yamls/flow-aggregator.yml b/build/yamls/flow-aggregator.yml
@@ -180,7 +180,7 @@ metadata:
   annotations: {}
   labels:
     app: flow-aggregator
-  name: flow-aggregator-configmap-2k727bgdf4
+  name: flow-aggregator-configmap-5df2dmbm8h
   namespace: flow-aggregator
 ---
 apiVersion: v1
@@ -248,7 +248,7 @@ spec:
       serviceAccountName: flow-aggregator
       volumes:
       - configMap:
-          name: flow-aggregator-configmap-2k727bgdf4
+          name: flow-aggregator-configmap-5df2dmbm8h
         name: flow-aggregator-config
       - hostPath:
           path: /var/log/antrea/flow-aggregator

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -136,6 +136,14 @@ func run(o *Options) error {
 	wireguardConfig := &config.WireGuardConfig{
 		Port: o.config.WireGuard.Port,
 	}
+	exceptCIDRs := []net.IPNet{}
+	for _, cidr := range o.config.Egress.ExceptCIDRs {
+		_, exceptCIDR, _ := net.ParseCIDR(cidr)
+		exceptCIDRs = append(exceptCIDRs, *exceptCIDR)
+	}
+	egressConfig := &config.EgressConfig{
+		ExceptCIDRs: exceptCIDRs,
+	}
 	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
@@ -175,6 +183,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		wireguardConfig,
+		egressConfig,
 		networkReadyCh,
 		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -176,6 +176,8 @@ type AgentConfig struct {
 	TransportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"`
 	// AntreaProxy contains AntreaProxy related configuration options.
 	AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"`
+	// Egress related configurations.
+	Egress EgressConfig `yaml:"egress"`
 }
 
 type AntreaProxyConfig struct {
@@ -196,3 +198,7 @@ type WireGuardConfig struct {
 	// The port for the WireGuard to receive traffic. Defaults to 51820.
 	Port int `yaml:"port,omitempty"`
 }
+
+type EgressConfig struct {
+	ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"`
+}
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -156,6 +156,14 @@ func (o *Options) validate(args []string) error {
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
+	if features.DefaultFeatureGate.Enabled(features.Egress) {
+		for _, cidr := range o.config.Egress.ExceptCIDRs {
+			_, _, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("Egress Except CIDR %s is invalid", cidr)
+			}
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -88,6 +88,7 @@ type Initializer struct {
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	wireGuardConfig *config.WireGuardConfig
+	egressConfig    *config.EgressConfig
 	enableProxy     bool
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
@@ -111,6 +112,7 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	wireGuardConfig *config.WireGuardConfig,
+	egressConfig *config.EgressConfig,
 	networkReadyCh chan<- struct{},
 	stopCh <-chan struct{},
 	enableProxy bool,
@@ -132,6 +134,7 @@ func NewInitializer(
 		serviceCIDRv6:         serviceCIDRv6,
 		networkConfig:         networkConfig,
 		wireGuardConfig:       wireGuardConfig,
+		egressConfig:          egressConfig,
 		networkReadyCh:        networkReadyCh,
 		stopCh:                stopCh,
 		enableProxy:           enableProxy,
@@ -385,7 +388,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 
 	// Install OpenFlow entries to enable Pod traffic to external IP
 	// addresses.
-	if err := i.ofClient.InstallExternalFlows(); err != nil {
+	if err := i.ofClient.InstallExternalFlows(i.egressConfig.ExceptCIDRs); err != nil {
 		klog.Errorf("Failed to install openflow entries for external connectivity: %v", err)
 		return err
 	}

diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go
@@ -95,6 +95,10 @@ type WireGuardConfig struct {
 	MTU int
 }
 
+type EgressConfig struct {
+	ExceptCIDRs []net.IPNet
+}
+
 // Local Node configurations retrieved from K8s API or host networking state.
 type NodeConfig struct {
 	// The Node's name used in Kubernetes.
@@ -130,6 +134,8 @@ type NodeConfig struct {
 	UplinkNetConfig *AdapterNetConfig
 	// The config of the WireGuard interface.
 	WireGuardConfig *WireGuardConfig
+	// The config of the Egress interface.
+	EgressConfig *EgressConfig
 }
 
 func (n *NodeConfig) String() string {