Introducing NodePortLocal in Antrea Agent (#1459)

This commit introduces NodePortLocal (NPL) in Antrea agent. With NPL, a Pod port can be directly reached from external network through a port of the Node on which the Pod is running. NPL programs IPTABLE rules to send incoming Node traffic to the target Pod port. This information is exposed through an annotation on the Pod object, for consumption by other entities, such as external load-balancers. An example of the pod annotation is given bellow: metadata: annotations: npl.k8s.io/endpoints: '[{"podport":"8080","nodeip":"10.102.47.229","nodeport":"40002"}]' To use this feature, following changes are required in the Antrea configuration: - Enable the "NodePortLocal" featureGate in the antrea-agent configuration - If desired, change the value of "nplPortRange" in the antrea-agent configuration When using this feature, the Antrea Agent need to mutate Pod objects, which is why access to the Pod API was added to the antrea-agent ClusterRole. Note: NPL agent is currently not supported in Windows platform Future changes: - Label based service filtering to select backend pods for NPL - Sync on start: compare programmed iptables rules with pod annotation during bootup and add/delete new rules in iptable only if required. Currently we are deleting all the rules and adding new rules for all the pods while booting up Antrea agent. - Add e2e test - Add documentation See #969 Signed-off-by: Hemant Shaw <[email protected]> Signed-off-by: Manu Dilip Shah <[email protected]> Signed-off-by: Monotosh Das <[email protected]> Signed-off-by: Shubham Chauhan <[email protected]> Signed-off-by: Sudipta Biswas <[email protected]>
antrea-io · Jan 12, 2021 · 90719c3 · 90719c3
1 parent 8fefd0c
commit 90719c3
Show file tree

Hide file tree

Showing 23 changed files with 1,501 additions and 19 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -869,6 +869,14 @@ rules:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - watch
+  - list
+  - update
+- apiGroups:
+  - ""
+  resources:
   - endpoints
   - services
   verbs:
@@ -1165,6 +1173,9 @@ data:
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
+    # Enable NodePortLocal feature to make the pods reachable externally through NodePort
+    #  NodePortLocal: false
+
     # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
     # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
     # feature that supports priorities, rule actions and externalEntities in the future.
@@ -1260,6 +1271,11 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+    # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+    # and all Node traffic directed to that port will be forwarded to the Pod.
+    #nplPortRange: 40000-41000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1316,7 +1332,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gm7dcbm584
+  name: antrea-config-b8hh7hm486
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1427,7 +1443,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gm7dcbm584
+          name: antrea-config-b8hh7hm486
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1692,7 +1708,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gm7dcbm584
+          name: antrea-config-b8hh7hm486
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -869,6 +869,14 @@ rules:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - watch
+  - list
+  - update
+- apiGroups:
+  - ""
+  resources:
   - endpoints
   - services
   verbs:
@@ -1165,6 +1173,9 @@ data:
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
+    # Enable NodePortLocal feature to make the pods reachable externally through NodePort
+    #  NodePortLocal: false
+
     # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
     # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
     # feature that supports priorities, rule actions and externalEntities in the future.
@@ -1260,6 +1271,11 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+    # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+    # and all Node traffic directed to that port will be forwarded to the Pod.
+    #nplPortRange: 40000-41000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1316,7 +1332,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-gm7dcbm584
+  name: antrea-config-b8hh7hm486
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1427,7 +1443,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-gm7dcbm584
+          name: antrea-config-b8hh7hm486
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1694,7 +1710,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-gm7dcbm584
+          name: antrea-config-b8hh7hm486
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -869,6 +869,14 @@ rules:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - watch
+  - list
+  - update
+- apiGroups:
+  - ""
+  resources:
   - endpoints
   - services
   verbs:
@@ -1165,6 +1173,9 @@ data:
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
+    # Enable NodePortLocal feature to make the pods reachable externally through NodePort
+    #  NodePortLocal: false
+
     # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
     # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
     # feature that supports priorities, rule actions and externalEntities in the future.
@@ -1260,6 +1271,11 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+    # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+    # and all Node traffic directed to that port will be forwarded to the Pod.
+    #nplPortRange: 40000-41000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1316,7 +1332,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-h7t8ffthht
+  name: antrea-config-hhfkgg2fg5
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1427,7 +1443,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-h7t8ffthht
+          name: antrea-config-hhfkgg2fg5
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1695,7 +1711,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-h7t8ffthht
+          name: antrea-config-hhfkgg2fg5
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -869,6 +869,14 @@ rules:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - watch
+  - list
+  - update
+- apiGroups:
+  - ""
+  resources:
   - endpoints
   - services
   verbs:
@@ -1165,6 +1173,9 @@ data:
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
+    # Enable NodePortLocal feature to make the pods reachable externally through NodePort
+    #  NodePortLocal: false
+
     # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
     # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
     # feature that supports priorities, rule actions and externalEntities in the future.
@@ -1265,6 +1276,11 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+    # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+    # and all Node traffic directed to that port will be forwarded to the Pod.
+    #nplPortRange: 40000-41000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1321,7 +1337,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mh52t2hmmd
+  name: antrea-config-bdc66g4872
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1441,7 +1457,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mh52t2hmmd
+          name: antrea-config-bdc66g4872
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1741,7 +1757,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mh52t2hmmd
+          name: antrea-config-bdc66g4872
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -869,6 +869,14 @@ rules:
   - ""
   resources:
   - pods
+  verbs:
+  - get
+  - watch
+  - list
+  - update
+- apiGroups:
+  - ""
+  resources:
   - endpoints
   - services
   verbs:
@@ -1165,6 +1173,9 @@ data:
     # Enable traceflow which provides packet tracing feature to diagnose network issue.
     #  Traceflow: true
 
+    # Enable NodePortLocal feature to make the pods reachable externally through NodePort
+    #  NodePortLocal: false
+
     # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
     # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
     # feature that supports priorities, rule actions and externalEntities in the future.
@@ -1265,6 +1276,11 @@ data:
     # the flow collector.
     # Flow export frequency should be greater than or equal to 1.
     #flowExportFrequency: 12
+
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+    # whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+    # and all Node traffic directed to that port will be forwarded to the Pod.
+    #nplPortRange: 40000-41000
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -1321,7 +1337,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mfd9dcdh6d
+  name: antrea-config-9964gfgbb4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -1432,7 +1448,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mfd9dcdh6d
+          name: antrea-config-9964gfgbb4
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -1697,7 +1713,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mfd9dcdh6d
+          name: antrea-config-9964gfgbb4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml
@@ -22,6 +22,14 @@ rules:
       - ""
     resources:
       - pods
+    verbs:
+      - get
+      - watch
+      - list
+      - update
+  - apiGroups:
+      - ""
+    resources:
       - endpoints
       - services
     verbs:

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -8,6 +8,9 @@ featureGates:
 # Enable traceflow which provides packet tracing feature to diagnose network issue.
 #  Traceflow: true
 
+# Enable NodePortLocal feature to make the pods reachable externally through NodePort
+#  NodePortLocal: false
+
 # Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
 # to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
 # feature that supports priorities, rule actions and externalEntities in the future.
@@ -108,3 +111,8 @@ featureGates:
 # the flow collector.
 # Flow export frequency should be greater than or equal to 1.
 #flowExportFrequency: 12
+
+# Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+# whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+# and all Node traffic directed to that port will be forwarded to the Pod.
+#nplPortRange: 40000-41000
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -36,6 +36,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/agent/flowexporter/flowrecords"
 	"github.com/vmware-tanzu/antrea/pkg/agent/interfacestore"
 	"github.com/vmware-tanzu/antrea/pkg/agent/metrics"
+	npl "github.com/vmware-tanzu/antrea/pkg/agent/nodeportlocal"
 	"github.com/vmware-tanzu/antrea/pkg/agent/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/agent/proxy"
 	"github.com/vmware-tanzu/antrea/pkg/agent/querier"
@@ -241,6 +242,15 @@ func run(o *Options) error {
 	// exits, we will force exit.
 	stopCh := signals.RegisterSignalHandlers()
 
+	// Start the NPL agent.
+	if features.DefaultFeatureGate.Enabled(features.NodePortLocal) {
+		nplController, err := npl.InitializeNPLAgent(k8sClient, o.config.NPLPortRange, nodeConfig.Name)
+		if err != nil {
+			return fmt.Errorf("failed to start NPL agent: %v", err)
+		}
+		go nplController.Run(stopCh)
+	}
+
 	log.StartLogFileNumberMonitor(stopCh)
 
 	go cniServer.Run(stopCh)

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -120,4 +120,8 @@ type AgentConfig struct {
 	// Flow export frequency should be greater than or equal to 1.
 	// Defaults to "12".
 	FlowExportFrequency uint `yaml:"flowExportFrequency,omitempty"`
+	// Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port from that range will be assigned
+	// whenever a Pod's container defines a specific port to be exposed (each container can define a list of ports as pod.spec.containers[].ports),
+	// and all Node traffic directed to that port will be forwarded to the Pod.
+	NPLPortRange string `yaml:"nplPortRange,omitempty"`
 }