Skip to content

Commit

Permalink
[ExternalNode] Handle ExternalNode from Antrea agent side (#3799)
Browse files Browse the repository at this point in the history
1. Provide an example RBAC yaml file for Antrea agent running
on VM with definitions of ClusterRole, ServiceAccount and
ClusterRoleBinding.

2. Add ExternalNodeController to monitor ExternalNode CRUD,
invoke interfaces to operate OVS and update interface store
with ExternalEntityInterface.

3. Implement OVS interactions related to ExternalNode CRUD.

4. Add a channel for receiving ExternalEntity updates from
ExternalNodeController and notifying NetworkPolicyController
to reconcile rules related to the updated ExternalEntities.
This is to handle the case when NetworkPolicyController reconciles
rules before ExternalEntityInterface is realized in the
interface store.

5. Support configuring policy bypass rules to skip ANP check.

Signed-off-by: Mengdie Song <[email protected]>
Co-authored-by: Wenying Dong <[email protected]>

Co-authored-by: Wenying Dong <[email protected]>
  • Loading branch information
mengdie-song and wenyingd authored Aug 2, 2022
1 parent f7c8b4b commit 18595a8
Show file tree
Hide file tree
Showing 36 changed files with 1,287 additions and 88 deletions.
3 changes: 0 additions & 3 deletions build/charts/antrea/conf/antrea-agent.conf
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ featureGates:
# Enable certificated-based authentication for IPsec.
{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }}

# Enable running agent on an unmanaged VM/BM.
{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ExternalNode" "default" false) }}

# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: {{ .Values.ovs.bridgeName | quote }}
Expand Down
7 changes: 2 additions & 5 deletions build/yamls/antrea-aks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2679,9 +2679,6 @@ data:
# Enable certificated-based authentication for IPsec.
# IPsecCertAuth: false
# Enable running agent on an unmanaged VM/BM.
# ExternalNode: false
# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: "br-int"
Expand Down Expand Up @@ -3780,7 +3777,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 34d282c5003d66a4a1741b1940b64ac6eb464275eed596ebf3a4242864bbb88a
checksum/config: e58a0311b8ecc3d02a5c5f9ab89a6d5e98beb2a7078f5cd36e6007bb860b1018
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4020,7 +4017,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 34d282c5003d66a4a1741b1940b64ac6eb464275eed596ebf3a4242864bbb88a
checksum/config: e58a0311b8ecc3d02a5c5f9ab89a6d5e98beb2a7078f5cd36e6007bb860b1018
labels:
app: antrea
component: antrea-controller
Expand Down
7 changes: 2 additions & 5 deletions build/yamls/antrea-eks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2679,9 +2679,6 @@ data:
# Enable certificated-based authentication for IPsec.
# IPsecCertAuth: false
# Enable running agent on an unmanaged VM/BM.
# ExternalNode: false
# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: "br-int"
Expand Down Expand Up @@ -3780,7 +3777,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 34d282c5003d66a4a1741b1940b64ac6eb464275eed596ebf3a4242864bbb88a
checksum/config: e58a0311b8ecc3d02a5c5f9ab89a6d5e98beb2a7078f5cd36e6007bb860b1018
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4022,7 +4019,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 34d282c5003d66a4a1741b1940b64ac6eb464275eed596ebf3a4242864bbb88a
checksum/config: e58a0311b8ecc3d02a5c5f9ab89a6d5e98beb2a7078f5cd36e6007bb860b1018
labels:
app: antrea
component: antrea-controller
Expand Down
7 changes: 2 additions & 5 deletions build/yamls/antrea-gke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2679,9 +2679,6 @@ data:
# Enable certificated-based authentication for IPsec.
# IPsecCertAuth: false
# Enable running agent on an unmanaged VM/BM.
# ExternalNode: false
# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: "br-int"
Expand Down Expand Up @@ -3780,7 +3777,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 9b7041970d4bf7d5bbf30003b5061a7f5ec2afed8c23c02d28a59e7a22805423
checksum/config: 8287f6d4c3b3def5067c65e1497df876878161a0b519b6d782298aa27356aab3
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4020,7 +4017,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 9b7041970d4bf7d5bbf30003b5061a7f5ec2afed8c23c02d28a59e7a22805423
checksum/config: 8287f6d4c3b3def5067c65e1497df876878161a0b519b6d782298aa27356aab3
labels:
app: antrea
component: antrea-controller
Expand Down
7 changes: 2 additions & 5 deletions build/yamls/antrea-ipsec.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2692,9 +2692,6 @@ data:
# Enable certificated-based authentication for IPsec.
# IPsecCertAuth: false
# Enable running agent on an unmanaged VM/BM.
# ExternalNode: false
# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: "br-int"
Expand Down Expand Up @@ -3793,7 +3790,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 94b202753c2ed0a7e187c486ccfbeb094e05ae7ee1f7001afb55e7c45eeeaad3
checksum/config: c216eb3adc199d8575af41ff151ae1b566381018527dd31a46d5efbcc4c0bde6
checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
labels:
app: antrea
Expand Down Expand Up @@ -4079,7 +4076,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 94b202753c2ed0a7e187c486ccfbeb094e05ae7ee1f7001afb55e7c45eeeaad3
checksum/config: c216eb3adc199d8575af41ff151ae1b566381018527dd31a46d5efbcc4c0bde6
labels:
app: antrea
component: antrea-controller
Expand Down
7 changes: 2 additions & 5 deletions build/yamls/antrea.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2679,9 +2679,6 @@ data:
# Enable certificated-based authentication for IPsec.
# IPsecCertAuth: false
# Enable running agent on an unmanaged VM/BM.
# ExternalNode: false
# Name of the OpenVSwitch bridge antrea-agent will create and use.
# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
ovsBridge: "br-int"
Expand Down Expand Up @@ -3780,7 +3777,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 17fb4f0ed9e653e3a470f41f980d2ff89a317686913f62195b6af62869779824
checksum/config: 745551965d4087e4a0e9854549a6d96472b6eeb12c269a432ea1ae6c873c028a
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4020,7 +4017,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 17fb4f0ed9e653e3a470f41f980d2ff89a317686913f62195b6af62869779824
checksum/config: 745551965d4087e4a0e9854549a6d96472b6eeb12c269a432ea1ae6c873c028a
labels:
app: antrea
component: antrea-controller
Expand Down
16 changes: 16 additions & 0 deletions build/yamls/externalnode/conf/antrea-agent.conf
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,22 @@ featureGates:
# Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode".
nodeType: externalNode

externalNode:
# The expected Namespace in which the ExternalNode is created.
# Defaults to "default".
#externalNodeNamespace: default

# The policyBypassRules describes the traffic that is expected to bypass NetworkPolicy rules.
# Each rule contains the following four attributes:
# direction (ingress|egress), protocol(tcp/udp/icmp/ip), remote CIDR, dst port (ICMP doesn't require).
# Here is an example:
# - direction: ingress
# protocol: tcp
# cidr: 1.1.1.1/32
# port: 22
# It is used only when NodeType is externalNode.
#policyBypassRules: []

# The path to access the kubeconfig file used in the connection to K8s APIServer. The file contains the K8s
# APIServer endpoint and the token of ServiceAccount required in the connection.
clientConnection:
Expand Down
112 changes: 112 additions & 0 deletions build/yamls/externalnode/vm-agent-rbac.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: vm-agent
namespace: vm-ns # Change the Namespace to where vm-agent is expected to run.
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vm-agent
rules:
# antrea-controller distributes the CA certificate as a ConfigMap named `antrea-ca` in the Antrea deployment Namespace.
# vm-agent needs to access `antrea-ca` to connect with antrea-controller.
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
- antrea-ca
verbs:
- get
- watch
- list
# This is the content of built-in role kube-system/extension-apiserver-authentication-reader.
# But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (vm-agent) will
# have permission issue after bumping up apiserver library to a version that supports dynamic authentication.
# See https://github.com/kubernetes/kubernetes/pull/85375
# To support K8s clusters older than v1.17.0, we grant the required permissions directly instead of relying on
# the extension-apiserver-authentication role.
- apiGroups:
- ""
resourceNames:
- extension-apiserver-authentication
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- crd.antrea.io
resources:
- antreaagentinfos
verbs:
- get
- update
- apiGroups:
- controlplane.antrea.io
resources:
- networkpolicies
- appliedtogroups
- addressgroups
verbs:
- get
- watch
- list
- apiGroups:
- controlplane.antrea.io
resources:
- nodestatssummaries
verbs:
- create
- apiGroups:
- controlplane.antrea.io
resources:
- networkpolicies/status
verbs:
- create
- get
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vm-agent
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: vm-agent
subjects:
- kind: ServiceAccount
name: vm-agent
namespace: vm-ns # Change the Namespace to where vm-agent is expected to run.
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vm-agent
namespace: vm-ns # Change the Namespace to where vm-agent is expected to run.
rules:
- apiGroups:
- crd.antrea.io
resources:
- externalnodes
verbs:
- get
- watch
- list
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vm-agent
namespace: vm-ns # Change the Namespace to where vm-agent is expected to run.
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: vm-agent
subjects:
- kind: ServiceAccount
name: vm-agent
namespace: vm-ns # Change the Namespace to where vm-agent is expected to run.
Loading

0 comments on commit 18595a8

Please sign in to comment.