Multicluster dataplane change for Service access (#3603)

* Add a new feature gate `Multicluster` and configs in antrea-agent.conf, and a few extra items in antrea-agent cluster role including access to `Gateway` and `ClusterInfoImport`. * Rename the `ServiceMarkTable` to `SNATMarkTable`. * Add a controller for Gateway Nodes to watch Gateway and ClusterInfoImport's events. It will set up a few openflow rules to forward cross-cluster traffic to remote Gateway Nodes. * Add a classification rule for cross-cluster traffic with global multicluster virtual MAC `aa:bb:cc:dd:ee:f0`. * Add a rule in `L3Forwarding` table for cross-cluster request packets that modifies the destination MAC to global multicluster virtual MAC. * Add a rule in `L3Forwarding` table for cross-cluster reply packets. * Add a rule to `SNATMark` table to match the packets of multi-cluster Service connection and perform DNAT in DNAT zone. * Add a rule to `SNAT` table to perform SNAT for any remote cluster traffic. * Add a rule to `UnSNAT` table to perform de-SNAT if destination IP is local GatewayIP. * Add a rule in `L2ForwardingCalc` table to load the global virtual multi-cluster MAC's output to `antrea-tun0`. * Add a rule in `Output` table to match the multi-cluster traffic to forward the traffic from/to regular Node through the same port. * Add a controller for regular Nodes to watch Gateway and ClusterInfoImport's events. It will set up a few openflow rules to forward cross-cluster traffic to local Gateway Node. * Add a rule in L3Forwarding table for cross-cluster request packets, and modify the destination MAC to global multicluster virtual MAC. * Add a rule in L3Forwarding table for cross-cluster reply packets. * Add a rule in L2ForwardingCalc table to load the global virtual multi-cluster MAC's output to `antrea-tun0`. * Use Service ClusterIPs instead of Pod IPs as MC Service's Endpoints. The ServiceExport controller will only watch ServiceExport and Service events, and wrap Service's ClusterIPs into a new Endpoint kind of ResourceExport. * Include local Service ClusterIP as multi-cluster Service's Endpoints as well. * Add unit test cases * Refine e2e test for data plane change Signed-off-by: Lan Luo <[email protected]> Co-authored-by: Hongliang Liu <[email protected]>
antrea-io · Jun 9, 2022 · 968b330 · 968b330
1 parent 5a7e5c3
commit 968b330
Show file tree

Hide file tree

Showing 50 changed files with 1,738 additions and 444 deletions.
diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md
@@ -82,6 +82,8 @@ Kubernetes: `>= 1.16.0-0`
 | logVerbosity | int | `0` |  |
 | multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". |
 | multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. |
+| multicluster.enable | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. |
+| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is antrea-agent's Namespace. |
 | noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. |
 | nodeIPAM.clusterCIDRs | list | `[]` | CIDR ranges to use when allocating Pod IP addresses. |
 | nodeIPAM.enable | bool | `false` | Enable Node IPAM in Antrea |

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -39,6 +39,10 @@ featureGates:
 # Enable multicast traffic. This feature is supported only with noEncap mode.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Multicast" "default" false) }}
 
+# Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+# This feature is supported only with encap mode.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Multicluster" "default" false) }}
+
 # Enable support for provisioning secondary network interfaces for Pods (using
 # Pod annotations). At the moment, Antrea can only create secondary network
 # interfaces using SR-IOV VFs on baremetal Nodes.
@@ -292,3 +296,13 @@ ipsec:
   #                  feature gate to be enabled.
   authenticationMode: {{ .authenticationMode | quote }}
 {{- end }}
+
+multicluster:
+{{- with .Values.multicluster }}
+# Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+# This feature is supported only with encap mode.
+  enable: {{ .enable }}
+# The Namespace where Antrea Multi-cluster Controller is running.
+# The default is antrea-agent's Namespace.
+  namespace: {{ .namespace | quote }}
+{{- end }}
diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml
@@ -189,3 +189,19 @@ rules:
       - watch
       - list
       - create
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - gateways
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - clusterinfoimports
+    verbs:
+    - get
+    - list
+    - watch
diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -281,6 +281,15 @@ logVerbosity: 0
 whereabouts:
   enable: false
 
+## -- Configure Multicluster, for use by the antrea-agent.
+multicluster:
+  # -- Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+  # This feature is supported only with encap mode.
+  enable: false
+  # -- The Namespace where Antrea Multi-cluster Controller is running.
+  # The default is antrea-agent's Namespace.
+  namespace: ""
+
 testing:
   ## -- enable code coverage measurement (used when testing Antrea only).
   coverage: false

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -97,6 +97,10 @@ data:
     # Enable multicast traffic. This feature is supported only with noEncap mode.
     #  Multicast: false
 
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+    #  Multicluster: false
+
     # Enable support for provisioning secondary network interfaces for Pods (using
     # Pod annotations). At the moment, Antrea can only create secondary network
     # interfaces using SR-IOV VFs on baremetal Nodes.
@@ -323,6 +327,14 @@ data:
       # - cert:          Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth`
       #                  feature gate to be enabled.
       authenticationMode: "psk"
+
+    multicluster:
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+      enable: false
+    # The Namespace where Antrea Multi-cluster Controller is running.
+    # The default is antrea-agent's Namespace.
+      namespace: ""
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3089,6 +3101,22 @@ rules:
       - watch
       - list
       - create
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - gateways
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - clusterinfoimports
+    verbs:
+    - get
+    - list
+    - watch
 ---
 # Source: antrea/templates/antctl/clusterrole.yaml
 kind: ClusterRole
@@ -3625,7 +3653,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb
+        checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f
       labels:
         app: antrea
         component: antrea-agent
@@ -3865,7 +3893,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb
+        checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -97,6 +97,10 @@ data:
     # Enable multicast traffic. This feature is supported only with noEncap mode.
     #  Multicast: false
 
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+    #  Multicluster: false
+
     # Enable support for provisioning secondary network interfaces for Pods (using
     # Pod annotations). At the moment, Antrea can only create secondary network
     # interfaces using SR-IOV VFs on baremetal Nodes.
@@ -323,6 +327,14 @@ data:
       # - cert:          Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth`
       #                  feature gate to be enabled.
       authenticationMode: "psk"
+
+    multicluster:
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+      enable: false
+    # The Namespace where Antrea Multi-cluster Controller is running.
+    # The default is antrea-agent's Namespace.
+      namespace: ""
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3089,6 +3101,22 @@ rules:
       - watch
       - list
       - create
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - gateways
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - clusterinfoimports
+    verbs:
+    - get
+    - list
+    - watch
 ---
 # Source: antrea/templates/antctl/clusterrole.yaml
 kind: ClusterRole
@@ -3625,7 +3653,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb
+        checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f
       labels:
         app: antrea
         component: antrea-agent
@@ -3867,7 +3895,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb
+        checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -97,6 +97,10 @@ data:
     # Enable multicast traffic. This feature is supported only with noEncap mode.
     #  Multicast: false
 
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+    #  Multicluster: false
+
     # Enable support for provisioning secondary network interfaces for Pods (using
     # Pod annotations). At the moment, Antrea can only create secondary network
     # interfaces using SR-IOV VFs on baremetal Nodes.
@@ -323,6 +327,14 @@ data:
       # - cert:          Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth`
       #                  feature gate to be enabled.
       authenticationMode: "psk"
+
+    multicluster:
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+      enable: false
+    # The Namespace where Antrea Multi-cluster Controller is running.
+    # The default is antrea-agent's Namespace.
+      namespace: ""
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3089,6 +3101,22 @@ rules:
       - watch
       - list
       - create
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - gateways
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - clusterinfoimports
+    verbs:
+    - get
+    - list
+    - watch
 ---
 # Source: antrea/templates/antctl/clusterrole.yaml
 kind: ClusterRole
@@ -3625,7 +3653,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6
+        checksum/config: 2f5a57b910bfb442df5abb3268308a3f4ad8f69d506e4281dddad39dab334690
       labels:
         app: antrea
         component: antrea-agent
@@ -3865,7 +3893,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6
+        checksum/config: 2f5a57b910bfb442df5abb3268308a3f4ad8f69d506e4281dddad39dab334690
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -110,6 +110,10 @@ data:
     # Enable multicast traffic. This feature is supported only with noEncap mode.
     #  Multicast: false
 
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+    #  Multicluster: false
+
     # Enable support for provisioning secondary network interfaces for Pods (using
     # Pod annotations). At the moment, Antrea can only create secondary network
     # interfaces using SR-IOV VFs on baremetal Nodes.
@@ -336,6 +340,14 @@ data:
       # - cert:          Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth`
       #                  feature gate to be enabled.
       authenticationMode: "psk"
+
+    multicluster:
+    # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic.
+    # This feature is supported only with encap mode.
+      enable: false
+    # The Namespace where Antrea Multi-cluster Controller is running.
+    # The default is antrea-agent's Namespace.
+      namespace: ""
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -3102,6 +3114,22 @@ rules:
       - watch
       - list
       - create
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - gateways
+    verbs:
+    - get
+    - list
+    - watch
+  - apiGroups:
+    - multicluster.crd.antrea.io
+    resources:
+    - clusterinfoimports
+    verbs:
+    - get
+    - list
+    - watch
 ---
 # Source: antrea/templates/antctl/clusterrole.yaml
 kind: ClusterRole
@@ -3638,7 +3666,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b
+        checksum/config: 009306e63cddc96c9dd51d20543783c6a94e9859581dc9db4dffacc8a78976bc
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -3924,7 +3952,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b
+        checksum/config: 009306e63cddc96c9dd51d20543783c6a94e9859581dc9db4dffacc8a78976bc
       labels:
         app: antrea
         component: antrea-controller