NodePort, LoadBalancer and ClusterIP from k8s Node support for Antrea…

…Proxy on Linux This PR implements: - The connection request of NodePort whose client is from remote or localhost. - The connection request of LoadBalancer whose client is from remote or localhost. - The connection request of ClusterIP whose client is from localhost. For NodePort support, an ipset is used to store NodePort IP and port in an entry IP:port,protocol. There are iptables rules which is used to match the ipset as destination and do DNAT with a virtual Service IP. There is a routing entry which is used to route the packets whose destination is the virtual Service IP to Antrea gateway. For LoadBalancer support, a routing entry will be created for every ingress IP to route the packets from remote or localhost to Antrea gateway. For ClusterIP support, a routing entry is always used to route the packets to Antrea gateway. when new ClusterIP are created, the destination IP block of the routing entry will be become large to include the IP address of ClusterIP. To support the Service traffic of above cases, there are main changes of OVS pipeline. - Add table serviceConntrackCommitTable 106 to perform SNAT for Service traffic. - Modify table hairpinSNATTable ID from 106 to 108. - Modify table serviceHairpinTable ID from 29 to 23. - Add table serviceConntrackTable 24 to transform SNATed connnections. - Add table serviceClassifierTable 35 to classify the Service traffic. - Add table serviceConntrackCommitTable 106 to do SNAT for Service traffic. Signed-off-by: Hongliang Liu <[email protected]>
antrea-io · Aug 16, 2021 · 49d7e53 · 49d7e53
1 parent 95a167a
commit 49d7e53
Show file tree

Hide file tree

Showing 44 changed files with 2,784 additions and 443 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3810,6 +3810,14 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+    # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+    #antreaProxyFull: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3892,7 +3900,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gdgb98gmfd
+  name: antrea-config-mb96584t95
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3963,7 +3971,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gdgb98gmfd
+          value: antrea-config-mb96584t95
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4014,7 +4022,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-mb96584t95
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4295,7 +4303,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-mb96584t95
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3810,6 +3810,14 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+    # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+    #antreaProxyFull: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3892,7 +3900,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gdgb98gmfd
+  name: antrea-config-mb96584t95
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3963,7 +3971,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-gdgb98gmfd
+          value: antrea-config-mb96584t95
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4014,7 +4022,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-mb96584t95
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4297,7 +4305,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gdgb98gmfd
+          name: antrea-config-mb96584t95
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3810,6 +3810,14 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+    # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+    #antreaProxyFull: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3892,7 +3900,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dc9bfdb7f6
+  name: antrea-config-b2bft59f77
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3963,7 +3971,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dc9bfdb7f6
+          value: antrea-config-b2bft59f77
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4014,7 +4022,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dc9bfdb7f6
+          name: antrea-config-b2bft59f77
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4298,7 +4306,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-dc9bfdb7f6
+          name: antrea-config-b2bft59f77
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3815,6 +3815,14 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+    # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+    #antreaProxyFull: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3897,7 +3905,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-cccf8b2ggf
+  name: antrea-config-8c848986g8
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3977,7 +3985,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-cccf8b2ggf
+          value: antrea-config-8c848986g8
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4028,7 +4036,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-cccf8b2ggf
+          name: antrea-config-8c848986g8
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4344,7 +4352,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-cccf8b2ggf
+          name: antrea-config-8c848986g8
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3815,6 +3815,14 @@ data:
     # If there are multiple IP addresses configured on the interface, the first one is used.
     # The interface configured with Node IP is used if this parameter is not set.
     #transportInterface:
+
+    # A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+    # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+    #nodePortAddresses: []
+
+    # Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+    # KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+    #antreaProxyFull: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3897,7 +3905,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mdd77fmc97
+  name: antrea-config-c88c4k7kt5
   namespace: kube-system
 ---
 apiVersion: v1
@@ -3968,7 +3976,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-mdd77fmc97
+          value: antrea-config-c88c4k7kt5
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4019,7 +4027,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mdd77fmc97
+          name: antrea-config-c88c4k7kt5
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4300,7 +4308,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mdd77fmc97
+          name: antrea-config-c88c4k7kt5
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -152,3 +152,11 @@ featureGates:
 # If there are multiple IP addresses configured on the interface, the first one is used.
 # The interface configured with Node IP is used if this parameter is not set.
 #transportInterface:
+
+# A string array of values which specifies the host IPv4/IPv6 addresses for NodePort. Values can be valid IP blocks.
+# (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+#nodePortAddresses: []
+
+# Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+# KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+#antreaProxyFull: false
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -104,7 +104,8 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		features.DefaultFeatureGate.Enabled(features.Egress),
-		features.DefaultFeatureGate.Enabled(features.FlowExporter))
+		features.DefaultFeatureGate.Enabled(features.FlowExporter),
+		o.config.AntreaProxyFull)
 
 	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
 	var serviceCIDRNetv6 *net.IPNet
@@ -121,7 +122,7 @@ func run(o *Options) error {
 		TransportIface:    o.config.TransportInterface,
 	}
 
-	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
+	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxyFull)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
 	}
@@ -172,13 +173,28 @@ func run(o *Options) error {
 	if features.DefaultFeatureGate.Enabled(features.AntreaProxy) {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
+		proxyFull := o.config.AntreaProxyFull
+		var nodePortIPv4Map, nodePortIPv6Map map[int][]net.IP
+		if proxyFull {
+			nodePortIPv4Map, nodePortIPv6Map, err = getAvailableNodePortIPs(o.config.NodePortAddresses, o.config.HostGateway)
+			if err != nil {
+				return fmt.Errorf("getting available NodePort IP addresses failed: %v", err)
+			}
+			if v4Enabled && len(nodePortIPv4Map) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv4 addresses was found")
+			}
+			if v6Enabled && len(nodePortIPv6Map) == 0 {
+				return fmt.Errorf("no qualified NodePort IPv6 addresses was found")
+			}
+		}
+
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient)
+			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortIPv4Map, nodePortIPv6Map, proxyFull)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortIPv4Map, proxyFull)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortIPv6Map, proxyFull)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -152,4 +152,10 @@ type AgentConfig struct {
 	// If there are multiple IP addresses configured on the interface, the first one is used.
 	// The interface configured with Node IP is used if this parameter is not set.
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// A string array of values which specifies the host IPv4/IPv6 addresses for NodePorts. Values may be valid IP blocks.
+	// (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
+	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
+	// Whether or not to enable full Service support in AntreaProxy in antrea-agent. If this option is true, then without
+	// KubeProxy, NodePort/LoadBalancer are supported, and ClusterIP can be accessed from host.
+	AntreaProxyFull bool `yaml:"antreaProxyFull,omitempty"`
 }
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -146,6 +146,9 @@ func (o *Options) validate(args []string) error {
 		// (but SNAT can be done by the primary CNI).
 		o.config.NoSNAT = true
 	}
+	if err := o.validateAntreaProxyConfig(); err != nil {
+		return fmt.Errorf("proxy config is invalid: %w", err)
+	}
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
@@ -219,6 +222,17 @@ func (o *Options) setDefaults() {
 	}
 }
 
+func (o *Options) validateAntreaProxyConfig() error {
+	if o.config.AntreaProxyFull {
+		for _, nodePortIP := range o.config.NodePortAddresses {
+			if _, _, err := net.ParseCIDR(nodePortIP); err != nil {
+				return fmt.Errorf("invalid NodePort IP address `%s`: %w", nodePortIP, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (o *Options) validateFlowExporterConfig() error {
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		host, port, proto, err := flowexport.ParseFlowCollectorAddr(o.config.FlowCollectorAddr, defaultFlowCollectorPort, defaultFlowCollectorTransport)