NodePort, LoadBalancer and ClusterIP from k8s Node support for Antrea…

…Proxy on Linux This PR implements: - The connection request of NodePort whose client is from remote or local host. - The connection request of LoadBalancer whose client is from remote or localhost. - The connection request of ClusterIP whose client is from localhost. For NodePort support, an ipset is used to store NodePort IP and port in an entry IP:port,protocol. The iptables rules are used to match the ipset as destination and do DNAT with a virtual IP. A routing entry is used to route the packets whose destination is the virtual IP to Antrea gateway. For LoadBalancer support, a routing entry will be created for every ingress IP to route the packets from remote or localhost to Antrea gateway. For ClusterIP support, a routing entry is always used to route the packets to Antrea gateway. when a new ClusterIP is created, the destination IP block of the routing entry might be extended to include the ClusterIP address To support the Service traffic of above cases, the main changes of OVS pipeline include: - Add table serviceConntrackCommitTable 106 to perform SNAT for Service traffic. - Modify table hairpinSNATTable ID from 106 to 108. - Modify table serviceHairpinTable ID from 29 to 23. - Add table serviceConntrackTable 24 to transform SNATed connnections. - Add table serviceClassifierTable 35 to classify the Service traffic. - Add table serviceConntrackCommitTable 106 to do SNAT for Service traffic. Signed-off-by: Hongliang Liu <[email protected]>
antrea-io · Aug 22, 2021 · e7c2b4c · e7c2b4c
1 parent 4077f80
commit e7c2b4c
Show file tree

Hide file tree

Showing 44 changed files with 2,568 additions and 461 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3901,6 +3901,16 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3983,7 +3993,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-66dt98cgtb
+  name: antrea-config-7fff2cg2ct
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4054,7 +4064,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-66dt98cgtb
+          value: antrea-config-7fff2cg2ct
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4105,7 +4115,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-66dt98cgtb
+          name: antrea-config-7fff2cg2ct
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4386,7 +4396,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-66dt98cgtb
+          name: antrea-config-7fff2cg2ct
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3901,6 +3901,16 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3983,7 +3993,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-66dt98cgtb
+  name: antrea-config-7fff2cg2ct
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4054,7 +4064,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-66dt98cgtb
+          value: antrea-config-7fff2cg2ct
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4105,7 +4115,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-66dt98cgtb
+          name: antrea-config-7fff2cg2ct
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4388,7 +4398,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-66dt98cgtb
+          name: antrea-config-7fff2cg2ct
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3901,6 +3901,16 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3983,7 +3993,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-d2f597tg62
+  name: antrea-config-k75c68ffg8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4054,7 +4064,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-d2f597tg62
+          value: antrea-config-k75c68ffg8
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4105,7 +4115,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-d2f597tg62
+          name: antrea-config-k75c68ffg8
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4389,7 +4399,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-d2f597tg62
+          name: antrea-config-k75c68ffg8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3906,6 +3906,16 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3988,7 +3998,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-bgd79km9c8
+  name: antrea-config-c658c8mdmh
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4068,7 +4078,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-bgd79km9c8
+          value: antrea-config-c658c8mdmh
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4119,7 +4129,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bgd79km9c8
+          name: antrea-config-c658c8mdmh
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4435,7 +4445,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-bgd79km9c8
+          name: antrea-config-c658c8mdmh
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3906,6 +3906,16 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # Provide the option to enable all Service support of AntreaProxy.
+    antreaProxy:
+      # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+      # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+      #proxyAll: false
+      # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+      # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+      # Note that, the option is only valid when proxyAll is true.
+      #nodePortAddresses: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3988,7 +3998,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dd8ffc8tk9
+  name: antrea-config-66k582hf94
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4059,7 +4069,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dd8ffc8tk9
+          value: antrea-config-66k582hf94
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4110,7 +4120,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dd8ffc8tk9
+          name: antrea-config-66k582hf94
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4391,7 +4401,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-dd8ffc8tk9
+          name: antrea-config-66k582hf94
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -151,3 +151,13 @@ featureGates:
 # If there are multiple IP addresses configured on the interface, the first one is used.
 # The interface configured with Node IP is used if this parameter is not set.
 #transportInterface:
+
+# Provide the option to enable all Service support of AntreaProxy.
+antreaProxy:
+  # Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+  # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+  #proxyAll: false
+  # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+  # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+  # Note that, the option is only valid when proxyAll is true.
+  #nodePortAddresses: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -104,7 +104,8 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		features.DefaultFeatureGate.Enabled(features.Egress),
-		features.DefaultFeatureGate.Enabled(features.FlowExporter))
+		features.DefaultFeatureGate.Enabled(features.FlowExporter),
+		o.config.AntreaProxy.ProxyAll)
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
 	var serviceCIDRNetv6 *net.IPNet
@@ -121,7 +122,7 @@ func run(o *Options) error {
 		TransportIface:    o.config.TransportInterface,
 	}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -172,13 +173,28 @@ func run(o *Options) error {
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		proxyAll := o.config.AntreaProxy.ProxyAll
+		var nodePortAddressesIPv4, nodePortAddressesIPv6 []net.IP
+		if proxyAll {
+			nodePortAddressesIPv4, nodePortAddressesIPv6, err = getAvailableNodePortAddresses(o.config.AntreaProxy.NodePortAddresses)
+			if err != nil {
+				return fmt.Errorf("getting available NodePort IP addresses failed: %v", err)
+			}
+			if v4Enabled && len(nodePortAddressesIPv4) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv4 addresses was found")
+			}
+			if v6Enabled && len(nodePortAddressesIPv6) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv6 addresses was found")
+			}
+		}
+
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -154,4 +154,13 @@ type AgentConfig struct {
 	// If there are multiple IP addresses configured on the interface, the first one is used.
 	// The interface configured with Node IP is used if this parameter is not set.
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// Provide the option to enable all Service support of AntreaProxy.
+	AntreaProxy struct {
+		// Whether to enable all Service support in AntreaProxy in antrea-agent. If this option is true, then without
+		// KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+		ProxyAll bool `yaml:"proxyAll,omitempty"`
+		// A string array of values which specifies the host IPv4/IPv6 addresses for NodePorts. Values may be valid IP blocks.
+		// (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+		NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
+	} `yaml:"antreaProxy,omitempty"`
 }
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -146,6 +146,9 @@ func (o *Options) validate(args []string) error {
 		// (but SNAT can be done by the primary CNI).
 		o.config.NoSNAT = true
 	}
+	if err := o.validateAntreaProxyConfig(); err != nil {
+		return fmt.Errorf("proxy config is invalid: %w", err)
+	}
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
@@ -219,6 +222,17 @@ func (o *Options) setDefaults() {
 	}
 }
 
+func (o *Options) validateAntreaProxyConfig() error {
+	if o.config.AntreaProxy.ProxyAll {
+		for _, nodePortAddress := range o.config.AntreaProxy.NodePortAddresses {
+			if _, _, err := net.ParseCIDR(nodePortAddress); err != nil {
+				return fmt.Errorf("invalid NodePort IP address `%s`: %w", nodePortAddress, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Options) validateFlowExporterConfig() error {
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		host, port, proto, err := flowexport.ParseFlowCollectorAddr(o.config.FlowCollectorAddr, defaultFlowCollectorPort, defaultFlowCollectorTransport)