[ExternalNode] Handle ExternalNode from Antrea agent side

1. Provide an example RBAC yaml file for Antrea agent running on VM with definitions of ClusterRole, ServiceAccount and ClusterRoleBinding. 2. Add ExternalNodeController to monitor ExternalNode CRUD, invoke interfaces to operate OVS and update interface store with ExternalEntitiyInterface. 3. Implement OVS interactions related to ExternalNode CRUD. 4. Add a channel for receiving ExternalNode updates from ExternalNodeController and notifying NetworkPolicyController to reconcile rules related to the updated ExternalNodes. This is to handle the case when NetworkPolicyController reconciles rules before ExternalEntitityInterface is realized in the interface store. 5. Update NetworkPolicy reconciler to invoke GetInterfacesByEntity() and GetContainerInterfacesByPod() for ExternalEntity and Pod separately. Signed-off-by: Mengdie Song <[email protected]> Co-authored-by: Wenying Dong <[email protected]>
antrea-io · Jun 21, 2022 · 5557fdd · 5557fdd
1 parent cd93bd3
commit 5557fdd
Show file tree

Hide file tree

Showing 26 changed files with 1,301 additions and 55 deletions.
diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -53,9 +53,6 @@ featureGates:
 # Enable certificated-based authentication for IPsec.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }}
 
-# Enable running agent on an unmanaged VM/BM.
-{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ExternalNode" "default" false) }}
-
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -111,9 +111,6 @@ data:
     # Enable certificated-based authentication for IPsec.
     #  IPsecCertAuth: false
 
-    # Enable running agent on an unmanaged VM/BM.
-    #  ExternalNode: false
-
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3648,7 +3645,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: c008cffcecc9959862be5461e6967f4c49c0bcc06dd9926cacc0ef2a37e5111c
+        checksum/config: add862bdadbc70ea7e2e2c44fc60f52d9ee753d5d1bad4c6ed6fc158cfd29d12
       labels:
         app: antrea
         component: antrea-agent
@@ -3888,7 +3885,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: c008cffcecc9959862be5461e6967f4c49c0bcc06dd9926cacc0ef2a37e5111c
+        checksum/config: add862bdadbc70ea7e2e2c44fc60f52d9ee753d5d1bad4c6ed6fc158cfd29d12
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -111,9 +111,6 @@ data:
     # Enable certificated-based authentication for IPsec.
     #  IPsecCertAuth: false
 
-    # Enable running agent on an unmanaged VM/BM.
-    #  ExternalNode: false
-
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3648,7 +3645,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: c008cffcecc9959862be5461e6967f4c49c0bcc06dd9926cacc0ef2a37e5111c
+        checksum/config: add862bdadbc70ea7e2e2c44fc60f52d9ee753d5d1bad4c6ed6fc158cfd29d12
       labels:
         app: antrea
         component: antrea-agent
@@ -3890,7 +3887,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: c008cffcecc9959862be5461e6967f4c49c0bcc06dd9926cacc0ef2a37e5111c
+        checksum/config: add862bdadbc70ea7e2e2c44fc60f52d9ee753d5d1bad4c6ed6fc158cfd29d12
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -111,9 +111,6 @@ data:
     # Enable certificated-based authentication for IPsec.
     #  IPsecCertAuth: false
 
-    # Enable running agent on an unmanaged VM/BM.
-    #  ExternalNode: false
-
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3648,7 +3645,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7cbfbb91897e3ebdbb7914a2e6831cef47ef08e6104c00aca6984bf32bd0e022
+        checksum/config: d0d5d7f0e4e920f762944fdd78e439ec68af61926a68f7f7f49f8f1ab0ac7fa6
       labels:
         app: antrea
         component: antrea-agent
@@ -3888,7 +3885,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7cbfbb91897e3ebdbb7914a2e6831cef47ef08e6104c00aca6984bf32bd0e022
+        checksum/config: d0d5d7f0e4e920f762944fdd78e439ec68af61926a68f7f7f49f8f1ab0ac7fa6
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -124,9 +124,6 @@ data:
     # Enable certificated-based authentication for IPsec.
     #  IPsecCertAuth: false
 
-    # Enable running agent on an unmanaged VM/BM.
-    #  ExternalNode: false
-
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3661,7 +3658,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 32f6800db6e4eaf5f97ee1d2135b07b44a96fc83bbd8f846b7f8fb616414edc6
+        checksum/config: 50764285ac4a299fe772f50f3e5f8b1ff320285f61e0e907353805fe881a06d7
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -3947,7 +3944,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 32f6800db6e4eaf5f97ee1d2135b07b44a96fc83bbd8f846b7f8fb616414edc6
+        checksum/config: 50764285ac4a299fe772f50f3e5f8b1ff320285f61e0e907353805fe881a06d7
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -111,9 +111,6 @@ data:
     # Enable certificated-based authentication for IPsec.
     #  IPsecCertAuth: false
 
-    # Enable running agent on an unmanaged VM/BM.
-    #  ExternalNode: false
-
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -3648,7 +3645,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: d729415bf3a9dafaff0d537bbc7104b4a8e672959f7804e342ef25d586747e83
+        checksum/config: 18b98dbb7837e8da7cebaf3c0cb35758708fd2372bc02f73aca86e01539f93a2
       labels:
         app: antrea
         component: antrea-agent
@@ -3888,7 +3885,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: d729415bf3a9dafaff0d537bbc7104b4a8e672959f7804e342ef25d586747e83
+        checksum/config: 18b98dbb7837e8da7cebaf3c0cb35758708fd2372bc02f73aca86e01539f93a2
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/externalnode/conf/antrea-agent.conf b/build/yamls/externalnode/conf/antrea-agent.conf
@@ -32,6 +32,10 @@ featureGates:
 # Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode".
 nodeType: externalNode
 
+# Namespace is the expected namespace to create the ExternalNode for the VM/BM object.
+# Defaults to "default".
+#namespace: default
+
 # The path to access the kubeconfig file used in the connection to K8s APIServer. The file contains the K8s
 # APIServer endpoint and the token of ServiceAccount required in the connection.
 clientConnection:

diff --git a/build/yamls/externalnode/vm-agent-rbac.yml b/build/yamls/externalnode/vm-agent-rbac.yml
@@ -0,0 +1,101 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: vm-agent
+  namespace: test-ns # Change the namespace to where vm-agent is expected to run.
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vm-agent
+rules:
+  # antrea-controller distributes the CA certificate as a ConfigMap named `antrea-ca` in the Antrea deployment Namespace.
+  # vm-agent needs to access `antrea-ca` to connect with antrea-controller.
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - antrea-ca
+    verbs:
+      - get
+      - watch
+      - list
+  # This is the content of built-in role kube-system/extension-apiserver-authentication-reader.
+  # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (vm-agent) will
+  # have permission issue after bumping up apiserver library to a version that supports dynamic authentication.
+  # See https://github.com/kubernetes/kubernetes/pull/85375
+  # To support K8s clusters older than v1.17.0, we grant the required permissions directly instead of relying on
+  # the extension-apiserver-authentication role.
+  - apiGroups:
+      - ""
+    resourceNames:
+      - extension-apiserver-authentication
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - crd.antrea.io
+    resourceNames:
+      - VM1 # Change the ExternalNode name which vm-agent is expected to update.
+    resources:
+      - antreaagentinfos
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - controlplane.antrea.io
+    resources:
+      - networkpolicies
+      - appliedtogroups
+      - addressgroups
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - controlplane.antrea.io
+    resources:
+      - nodestatssummaries
+    verbs:
+      - create
+  - apiGroups:
+      - controlplane.antrea.io
+    resources:
+      - networkpolicies/status
+    verbs:
+      - create
+      - get
+  - apiGroups:
+      - crd.antrea.io
+    resources:
+      - externalentities
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups:
+      - crd.antrea.io
+    resources:
+      - externalnodes
+    verbs:
+      - get
+      - watch
+      - list
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vm-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: vm-agent
+subjects:
+  - kind: ServiceAccount
+    name: vm-agent
+    namespace: test-ns # Change the namespace to where vm-agent is expected to run.
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -40,6 +40,7 @@ import (
 	"antrea.io/antrea/pkg/agent/controller/noderoute"
 	"antrea.io/antrea/pkg/agent/controller/serviceexternalip"
 	"antrea.io/antrea/pkg/agent/controller/traceflow"
+	"antrea.io/antrea/pkg/agent/externalnode"
 	"antrea.io/antrea/pkg/agent/flowexporter"
 	"antrea.io/antrea/pkg/agent/flowexporter/exporter"
 	"antrea.io/antrea/pkg/agent/interfacestore"
@@ -101,6 +102,7 @@ func run(o *Options) error {
 	serviceInformer := informerFactory.Core().V1().Services()
 	endpointsInformer := informerFactory.Core().V1().Endpoints()
 	externalIPPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools()
+	externalNodeInformer := crdInformerFactory.Crd().V1alpha1().ExternalNodes()
 
 	// Create Antrea Clientset for the given config.
 	antreaClientProvider := agent.NewAntreaClientProvider(o.config.AntreaClientConnection, k8sClient)
@@ -216,6 +218,7 @@ func run(o *Options) error {
 	// Initialize agent and node network.
 	agentInitializer := agent.NewInitializer(
 		k8sClient,
+		crdClient,
 		ovsBridgeClient,
 		ofClient,
 		routeClient,
@@ -230,6 +233,7 @@ func run(o *Options) error {
 		networkReadyCh,
 		stopCh,
 		o.nodeType,
+		o.config.Namespace,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		o.config.AntreaProxy.ProxyAll,
 		connectUplinkToBridge)
@@ -296,7 +300,16 @@ func run(o *Options) error {
 	// podUpdateChannel is a channel for receiving Pod updates from CNIServer and
 	// notifying NetworkPolicyController and EgressController to reconcile rules
 	// related to the updated Pods.
-	podUpdateChannel := channel.NewSubscribableChannel("PodUpdate", 100)
+	var podUpdateChannel *channel.SubscribableChannel
+	// externalNodeUpdateChannel is a channel for receiving ExternalNode updates from ExternalNodeController and
+	// notifying NetworkPolicyController to reconcile rules related to the updated ExternalNodes.
+	var externalNodeUpdateChannel *channel.SubscribableChannel
+	if o.nodeType == config.K8sNode {
+		podUpdateChannel = channel.NewSubscribableChannel("PodUpdate", 100)
+	} else {
+		externalNodeUpdateChannel = channel.NewSubscribableChannel("ExternalNodeUpdate", 100)
+	}
+
 	// We set flow poll interval as the time interval for rule deletion in the async
 	// rule cache, which is implemented as part of the idAllocator. This is to preserve
 	// the rule info for populating NetworkPolicy fields in the Flow Exporter even
@@ -315,6 +328,7 @@ func run(o *Options) error {
 		ifaceStore,
 		nodeConfig.Name,
 		podUpdateChannel,
+		externalNodeUpdateChannel,
 		groupCounters,
 		groupIDUpdates,
 		antreaPolicyEnabled,
@@ -323,6 +337,7 @@ func run(o *Options) error {
 		loggingEnabled,
 		asyncRuleDeleteInterval,
 		o.config.DNSServerOverride,
+		o.nodeType,
 		v4Enabled,
 		v6Enabled)
 	if err != nil {
@@ -386,6 +401,7 @@ func run(o *Options) error {
 
 	var cniServer *cniserver.CNIServer
 	var cniPodInfoStore cnipodcache.CNIPodInfoStore
+	var externalNodeController *externalnode.ExternalNodeController
 	if o.nodeType == config.K8sNode {
 		isChaining := false
 		if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
@@ -415,6 +431,12 @@ func run(o *Options) error {
 				return fmt.Errorf("error initializing CNI server: %v", err)
 			}
 		}
+	} else {
+		externalNodeController, err = externalnode.NewExternalNodeController(ovsBridgeClient, ofClient, externalNodeInformer,
+			ifaceStore, externalNodeUpdateChannel, o.config.Namespace)
+		if err != nil {
+			return fmt.Errorf("error creating ExternalNode controller: %v", err)
+		}
 	}
 
 	var traceflowController *traceflow.Controller
@@ -507,6 +529,9 @@ func run(o *Options) error {
 		go podUpdateChannel.Run(stopCh)
 		go cniServer.Run(stopCh)
 		go nodeRouteController.Run(stopCh)
+	} else {
+		go externalNodeUpdateChannel.Run(stopCh)
+		go externalNodeController.Run(stopCh)
 	}
 
 	if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec &&

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -489,4 +489,7 @@ func (o *Options) setExternalNodeDefaultOptions() {
 		o.config.EnablePrometheusMetrics = new(bool)
 		*o.config.EnablePrometheusMetrics = false
 	}
+	if o.config.Namespace == "" {
+		o.config.Namespace = "default"
+	}
 }