NodePort, LoadBalancer and ClusterIP from k8s Node support for Antrea…

…Proxy on Linux This PR implements: - The connection request of NodePort whose client is from remote or local host. - The connection request of LoadBalancer whose client is from remote or localhost. - The connection request of ClusterIP whose client is from localhost. For NodePort support, an ipset is used to store NodePort IP and port in an entry IP:port,protocol. The iptables rules are used to match the ipset as destination and do DNAT with a virtual IP. A routing entry is used to route the packets whose destination is the virtual IP to Antrea gateway. For LoadBalancer support, a routing entry will be created for every ingress IP to route the packets from remote or localhost to Antrea gateway. For ClusterIP support, a routing entry is always used to route the packets to Antrea gateway. when a new ClusterIP is created, the destination IP block of the routing entry might be extended to include the ClusterIP address To support the Service traffic of above cases, the main changes of OVS pipeline include: - Add table serviceConntrackCommitTable 106 to perform SNAT for Service traffic. - Modify table hairpinSNATTable ID from 106 to 108. - Modify table serviceHairpinTable ID from 29 to 23. - Add table serviceConntrackTable 24 to transform SNATed connnections. - Add table serviceClassifierTable 35 to classify the Service traffic. - Add table serviceConntrackCommitTable 106 to do SNAT for Service traffic. Signed-off-by: Hongliang Liu <[email protected]>
antrea-io · Aug 19, 2021 · 3d67f4d · 3d67f4d
1 parent 43837c6
commit 3d67f4d
Show file tree

Hide file tree

Showing 44 changed files with 2,590 additions and 468 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3718,7 +3718,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  antrea-agent.conf: |
+  antrea-agent.conf: |-
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
@@ -3868,6 +3868,20 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3950,7 +3964,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gdgb98gmfd
+  name: antrea-config-cb24t9bh76
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4021,7 +4035,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gdgb98gmfd
+          value: antrea-config-cb24t9bh76
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4072,7 +4086,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-cb24t9bh76
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4353,7 +4367,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-cb24t9bh76
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3718,7 +3718,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  antrea-agent.conf: |
+  antrea-agent.conf: |-
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
@@ -3868,6 +3868,20 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3950,7 +3964,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gdgb98gmfd
+  name: antrea-config-cb24t9bh76
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4021,7 +4035,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gdgb98gmfd
+          value: antrea-config-cb24t9bh76
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4072,7 +4086,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-cb24t9bh76
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4355,7 +4369,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-cb24t9bh76
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3718,7 +3718,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  antrea-agent.conf: |
+  antrea-agent.conf: |-
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
@@ -3868,6 +3868,20 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3950,7 +3964,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dc9bfdb7f6
+  name: antrea-config-9cdtmgk75f
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4021,7 +4035,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dc9bfdb7f6
+          value: antrea-config-9cdtmgk75f
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4072,7 +4086,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dc9bfdb7f6
+          name: antrea-config-9cdtmgk75f
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4356,7 +4370,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-dc9bfdb7f6
+          name: antrea-config-9cdtmgk75f
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3718,7 +3718,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  antrea-agent.conf: |
+  antrea-agent.conf: |-
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
@@ -3873,6 +3873,20 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3955,7 +3969,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-cccf8b2ggf
+  name: antrea-config-d2tb47k2g9
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4035,7 +4049,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-cccf8b2ggf
+          value: antrea-config-d2tb47k2g9
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4086,7 +4100,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-cccf8b2ggf
+          name: antrea-config-d2tb47k2g9
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4402,7 +4416,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-cccf8b2ggf
+          name: antrea-config-d2tb47k2g9
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3718,7 +3718,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  antrea-agent.conf: |
+  antrea-agent.conf: |-
     # FeatureGates is a map of feature names to bools that enable or disable experimental features.
     featureGates:
     # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent.
@@ -3873,6 +3873,20 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3955,7 +3969,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mdd77fmc97
+  name: antrea-config-9kmkh7d7cg
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4026,7 +4040,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-mdd77fmc97
+          value: antrea-config-9kmkh7d7cg
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4077,7 +4091,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mdd77fmc97
+          name: antrea-config-9kmkh7d7cg
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4358,7 +4372,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mdd77fmc97
+          name: antrea-config-9kmkh7d7cg
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -152,3 +152,17 @@ featureGates:
 # If there are multiple IP addresses configured on the interface, the first one is used.
 # The interface configured with Node IP is used if this parameter is not set.
 #transportInterface:
+
+# A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+#nodePortAddresses: []
+
+# Provide the option to enable all Service support of AntreaProxy.
+antreaProxy:
+  # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+  # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+  #proxyAll: false
+  # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+  # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+  # Note that, the option is only valid when proxyAll is true.
+  #nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -104,7 +104,8 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		features.DefaultFeatureGate.Enabled(features.Egress),
-		features.DefaultFeatureGate.Enabled(features.FlowExporter))
+		features.DefaultFeatureGate.Enabled(features.FlowExporter),
+		o.config.AntreaProxy.ProxyAll)
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
 	var serviceCIDRNetv6 *net.IPNet
@@ -121,7 +122,7 @@ func run(o *Options) error {
 		TransportIface:    o.config.TransportInterface,
 	}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -172,13 +173,28 @@ func run(o *Options) error {
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		proxyAll := o.config.AntreaProxy.ProxyAll
+		var nodePortAddressesIPv4, nodePortAddressesIPv6 []net.IP
+		if proxyAll {
+			nodePortAddressesIPv4, nodePortAddressesIPv6, err = getAvailableNodePortAddresses(o.config.AntreaProxy.NodePortAddresses)
+			if err != nil {
+				return fmt.Errorf("getting available NodePort IP addresses failed: %v", err)
+			}
+			if v4Enabled && len(nodePortAddressesIPv4) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv4 addresses was found")
+			}
+			if v6Enabled && len(nodePortAddressesIPv6) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv6 addresses was found")
+			}
+		}
+
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}