Support NodeLocal DNSCache with AntreaProxy

Resolves #2137 Signed-off-by: Lan Luo <[email protected]>
antrea-io · Oct 12, 2021 · c1eaea1 · c1eaea1
1 parent 95e836d
commit c1eaea1
Show file tree

Hide file tree

Showing 13 changed files with 112 additions and 36 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3948,6 +3948,9 @@ data:
       # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
       # Note that the option is only valid when proxyAll is true.
       #nodePortAddresses: []
+      # A string array of values which specifies the service list should skip proxying. Values can be a valid
+      # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+      #skipServices: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4054,7 +4057,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b72h88gb7b
+  name: antrea-config-9mtb572bg4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4128,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-b72h88gb7b
+          value: antrea-config-9mtb572bg4
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4179,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b72h88gb7b
+          name: antrea-config-9mtb572bg4
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4457,7 +4460,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b72h88gb7b
+          name: antrea-config-9mtb572bg4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3948,6 +3948,9 @@ data:
       # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
       # Note that the option is only valid when proxyAll is true.
       #nodePortAddresses: []
+      # A string array of values which specifies the service list should skip proxying. Values can be a valid
+      # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+      #skipServices: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4054,7 +4057,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-b72h88gb7b
+  name: antrea-config-9mtb572bg4
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4128,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-b72h88gb7b
+          value: antrea-config-9mtb572bg4
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4179,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-b72h88gb7b
+          name: antrea-config-9mtb572bg4
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4459,7 +4462,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-b72h88gb7b
+          name: antrea-config-9mtb572bg4
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3948,6 +3948,9 @@ data:
       # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
       # Note that the option is only valid when proxyAll is true.
       #nodePortAddresses: []
+      # A string array of values which specifies the service list should skip proxying. Values can be a valid
+      # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+      #skipServices: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4054,7 +4057,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-hfkckg6t57
+  name: antrea-config-76b7d87725
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4128,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-hfkckg6t57
+          value: antrea-config-76b7d87725
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4179,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-hfkckg6t57
+          name: antrea-config-76b7d87725
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4460,7 +4463,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-hfkckg6t57
+          name: antrea-config-76b7d87725
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3953,6 +3953,9 @@ data:
       # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
       # Note that the option is only valid when proxyAll is true.
       #nodePortAddresses: []
+      # A string array of values which specifies the service list should skip proxying. Values can be a valid
+      # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+      #skipServices: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4059,7 +4062,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-4f28b82tdt
+  name: antrea-config-m542mcht2d
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4139,7 +4142,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-4f28b82tdt
+          value: antrea-config-m542mcht2d
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4190,7 +4193,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-4f28b82tdt
+          name: antrea-config-m542mcht2d
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4506,7 +4509,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-4f28b82tdt
+          name: antrea-config-m542mcht2d
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3953,6 +3953,9 @@ data:
       # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
       # Note that the option is only valid when proxyAll is true.
       #nodePortAddresses: []
+      # A string array of values which specifies the service list should skip proxying. Values can be a valid
+      # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+      #skipServices: []
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4059,7 +4062,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-bmthb2m52d
+  name: antrea-config-528c6hg6t7
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4130,7 +4133,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-bmthb2m52d
+          value: antrea-config-528c6hg6t7
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4181,7 +4184,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-bmthb2m52d
+          name: antrea-config-528c6hg6t7
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4462,7 +4465,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-bmthb2m52d
+          name: antrea-config-528c6hg6t7
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -190,3 +190,6 @@ antreaProxy:
   # (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
   # Note that the option is only valid when proxyAll is true.
   #nodePortAddresses: []
+  # A string array of values which specifies the service list should skip proxying. Values can be a valid
+  # ClusterIP (e.g. 10.11.1.2) or a Service name with Namespace (e.g. kube-system/kube-dns)
+  #skipServices: []
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -203,14 +203,15 @@ func run(o *Options) error {
 		v4Enabled := config.IsIPv4Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		v6Enabled := config.IsIPv6Enabled(nodeConfig, networkConfig.TrafficEncapMode)
 		proxyAll := o.config.AntreaProxy.ProxyAll
+		skipServices := o.config.AntreaProxy.SkipServices
 
 		switch {
 		case v4Enabled && v6Enabled:
-			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll)
+			proxier = proxy.NewDualStackProxier(nodeConfig.Name, informerFactory, ofClient, routeClient, nodePortAddressesIPv4, nodePortAddressesIPv6, proxyAll, skipServices)
 		case v4Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAll, skipServices)
 		case v6Enabled:
-			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll)
+			proxier = proxy.NewProxier(nodeConfig.Name, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAll, skipServices)
 		default:
 			return fmt.Errorf("at least one of IPv4 or IPv6 should be enabled")
 		}

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -186,6 +186,9 @@ type AntreaProxyConfig struct {
 	// A string array of values which specifies the host IPv4/IPv6 addresses for NodePorts. Values may be valid IP blocks.
 	// (e.g. 1.2.3.0/24, 1.2.3.4/32). An empty string slice is meant to select all host IPv4/IPv6 addresses.
 	NodePortAddresses []string `yaml:"nodePortAddresses,omitempty"`
+	// A string array of values which specifies the service list should skip proxying. Values may be valid ClusterIP (e.g. 10.11.1.2)
+	// or Service name with namespace (e.g. kube-system/kube-dns)
+	SkipServices []string `yaml:"skipServices,omitempty"`
 }
 
 type WireGuardConfig struct {

diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go
@@ -782,7 +782,8 @@ func NewProxier(
 	isIPv6 bool,
 	routeClient route.Interface,
 	nodePortAddresses []net.IP,
-	proxyAllEnabled bool) *proxier {
+	proxyAllEnabled bool,
+	skipServices []string) *proxier {
 	recorder := record.NewBroadcaster().NewRecorder(
 		runtime.NewScheme(),
 		corev1.EventSource{Component: componentName, Host: hostname},
@@ -800,7 +801,7 @@ func NewProxier(
 		endpointsConfig:          config.NewEndpointsConfig(informerFactory.Core().V1().Endpoints(), resyncPeriod),
 		serviceConfig:            config.NewServiceConfig(informerFactory.Core().V1().Services(), resyncPeriod),
 		endpointsChanges:         newEndpointsChangesTracker(hostname, endpointSliceEnabled, isIPv6),
-		serviceChanges:           newServiceChangesTracker(recorder, ipFamily),
+		serviceChanges:           newServiceChangesTracker(recorder, ipFamily, skipServices),
 		serviceMap:               k8sproxy.ServiceMap{},
 		serviceInstalledMap:      k8sproxy.ServiceMap{},
 		endpointsInstalledMap:    types.EndpointsMap{},
@@ -866,13 +867,14 @@ func NewDualStackProxier(
 	routeClient route.Interface,
 	nodePortAddressesIPv4 []net.IP,
 	nodePortAddressesIPv6 []net.IP,
-	proxyAllEnabled bool) *metaProxierWrapper {
+	proxyAllEnabled bool,
+	skipServices []string) *metaProxierWrapper {
 
 	// Create an IPv4 instance of the single-stack proxier.
-	ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled)
+	ipv4Proxier := NewProxier(hostname, informerFactory, ofClient, false, routeClient, nodePortAddressesIPv4, proxyAllEnabled, skipServices)
 
 	// Create an IPv6 instance of the single-stack proxier.
-	ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled)
+	ipv6Proxier := NewProxier(hostname, informerFactory, ofClient, true, routeClient, nodePortAddressesIPv6, proxyAllEnabled, skipServices)
 
 	// Create a meta-proxier that dispatch calls between the two
 	// single-stack proxier instances.

diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go
@@ -114,7 +114,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP
 
 	p := &proxier{
 		endpointsChanges:         newEndpointsChangesTracker(hostname, false, isIPv6),
-		serviceChanges:           newServiceChangesTracker(recorder, ipFamily),
+		serviceChanges:           newServiceChangesTracker(recorder, ipFamily, []string{"kube-system/kube-dns", "192.168.1.2"}),
 		serviceMap:               k8sproxy.ServiceMap{},
 		serviceInstalledMap:      k8sproxy.ServiceMap{},
 		endpointsInstalledMap:    types.EndpointsMap{},
@@ -145,6 +145,18 @@ func testClusterIP(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) {
 		Port:           "80",
 		Protocol:       corev1.ProtocolTCP,
 	}
+	svc1Port := 53
+	svc1PortName := k8sproxy.ServicePortName{
+		NamespacedName: makeNamespaceName("kube-system", "kube-dns"),
+		Port:           "53",
+		Protocol:       corev1.ProtocolTCP,
+	}
+	svc2Port := 88
+	svc2PortName := k8sproxy.ServicePortName{
+		NamespacedName: makeNamespaceName("kube-system", "test"),
+		Port:           "88",
+		Protocol:       corev1.ProtocolTCP,
+	}
 	makeServiceMap(fp,
 		makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *corev1.Service) {
 			svc.Spec.ClusterIP = svcIP.String()
@@ -154,6 +166,22 @@ func testClusterIP(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) {
 				Protocol: corev1.ProtocolTCP,
 			}}
 		}),
+		makeTestService(svc1PortName.Namespace, svc1PortName.Name, func(svc *corev1.Service) {
+			svc.Spec.ClusterIP = "10.96.10.12"
+			svc.Spec.Ports = []corev1.ServicePort{{
+				Name:     svc1PortName.Port,
+				Port:     int32(svc1Port),
+				Protocol: corev1.ProtocolTCP,
+			}}
+		}),
+		makeTestService(svc2PortName.Namespace, svc2PortName.Name, func(svc *corev1.Service) {
+			svc.Spec.ClusterIP = "192.168.1.2"
+			svc.Spec.Ports = []corev1.ServicePort{{
+				Name:     svc2PortName.Port,
+				Port:     int32(svc2Port),
+				Protocol: corev1.ProtocolTCP,
+			}}
+		}),
 	)
 
 	makeEndpointsMap(fp,

diff --git a/pkg/agent/proxy/service.go b/pkg/agent/proxy/service.go
@@ -31,8 +31,8 @@ type serviceChangesTracker struct {
 	initialized bool
 }
 
-func newServiceChangesTracker(recorder record.EventRecorder, ipFamily v1.IPFamily) *serviceChangesTracker {
-	return &serviceChangesTracker{tracker: k8sproxy.NewServiceChangeTracker(types.NewServiceInfo, ipFamily, recorder, nil)}
+func newServiceChangesTracker(recorder record.EventRecorder, ipFamily v1.IPFamily, skipServices []string) *serviceChangesTracker {
+	return &serviceChangesTracker{tracker: k8sproxy.NewServiceChangeTracker(types.NewServiceInfo, ipFamily, recorder, nil, skipServices)}
 }
 
 func (sh *serviceChangesTracker) OnServiceSynced() {