From 7de42660a04665c9be4ae9c83bc74b7fc331fa88 Mon Sep 17 00:00:00 2001 From: Zhecheng Date: Tue, 12 Jan 2021 22:01:56 +0800 Subject: [PATCH] Provide CipherSuites option between Clients and Servers * Add options TLSCipherSuites, TLSMinVersion on Apiserver side of controller and agent. * Add a util module cipher to support functionality and related UT. * Add tls e2e tests to verify Apiserver of Antrea and Antrea agent. Signed-off-by: Zhecheng Li --- build/yamls/antrea-aks.yml | 24 ++- build/yamls/antrea-eks.yml | 24 ++- build/yamls/antrea-gke.yml | 24 ++- build/yamls/antrea-ipsec.yml | 24 ++- build/yamls/antrea.yml | 24 ++- build/yamls/base/conf/antrea-agent.conf | 9 ++ build/yamls/base/conf/antrea-controller.conf | 9 ++ cmd/antrea-agent/agent.go | 9 +- cmd/antrea-agent/config.go | 4 + cmd/antrea-controller/config.go | 4 + cmd/antrea-controller/controller.go | 16 +- pkg/agent/apiserver/apiserver.go | 4 +- pkg/util/cipher/cipher.go | 42 ++++++ pkg/util/cipher/cipher_test.go | 52 +++++++ test/e2e/framework.go | 1 + test/e2e/tls_test.go | 151 +++++++++++++++++++ 16 files changed, 402 insertions(+), 19 deletions(-) create mode 100644 pkg/util/cipher/cipher.go create mode 100644 pkg/util/cipher/cipher_test.go create mode 100644 test/e2e/tls_test.go diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 9ad66112354..3e42a38a8f1 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1371,6 +1371,15 @@ data: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -1422,12 +1431,21 @@ data: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: kind: ConfigMap metadata: annotations: {} labels: app: antrea - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 namespace: kube-system --- apiVersion: v1 @@ -1538,7 +1556,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 name: antrea-config - name: antrea-controller-tls secret: @@ -1802,7 +1820,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index b2bcc8df1bb..0fc8670e980 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1371,6 +1371,15 @@ data: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -1422,12 +1431,21 @@ data: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: kind: ConfigMap metadata: annotations: {} labels: app: antrea - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 namespace: kube-system --- apiVersion: v1 @@ -1538,7 +1556,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 name: antrea-config - name: antrea-controller-tls secret: @@ -1804,7 +1822,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-7d66b472ff + name: antrea-config-gt6f55df69 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 664587c5ddb..f0f666463c2 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1371,6 +1371,15 @@ data: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -1422,12 +1431,21 @@ data: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: kind: ConfigMap metadata: annotations: {} labels: app: antrea - name: antrea-config-8hc5c7g4hb + name: antrea-config-56ghk45g94 namespace: kube-system --- apiVersion: v1 @@ -1538,7 +1556,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-8hc5c7g4hb + name: antrea-config-56ghk45g94 name: antrea-config - name: antrea-controller-tls secret: @@ -1805,7 +1823,7 @@ spec: path: /home/kubernetes/bin name: host-cni-bin - configMap: - name: antrea-config-8hc5c7g4hb + name: antrea-config-56ghk45g94 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 9094fc04954..6c87cdfcc42 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1376,6 +1376,15 @@ data: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -1427,12 +1436,21 @@ data: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: kind: ConfigMap metadata: annotations: {} labels: app: antrea - name: antrea-config-k7574f7tdc + name: antrea-config-c5f94kkkd9 namespace: kube-system --- apiVersion: v1 @@ -1552,7 +1570,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-k7574f7tdc + name: antrea-config-c5f94kkkd9 name: antrea-config - name: antrea-controller-tls secret: @@ -1851,7 +1869,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-k7574f7tdc + name: antrea-config-c5f94kkkd9 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index f92fdafee26..a918a354e63 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1376,6 +1376,15 @@ data: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -1427,12 +1436,21 @@ data: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + + # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. + # https://golang.org/pkg/crypto/tls/#pkg-constants + # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always + # prefer TLS1.3 Cipher Suites whenever possible. + #tlsCipherSuites: + + # TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. + #tlsMinVersion: kind: ConfigMap metadata: annotations: {} labels: app: antrea - name: antrea-config-2m4ktcghmf + name: antrea-config-6h4c4bttfd namespace: kube-system --- apiVersion: v1 @@ -1543,7 +1561,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-2m4ktcghmf + name: antrea-config-6h4c4bttfd name: antrea-config - name: antrea-controller-tls secret: @@ -1807,7 +1825,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-2m4ktcghmf + name: antrea-config-6h4c4bttfd name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index f03b6c6c000..2703afc6f75 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -120,3 +120,12 @@ featureGates: # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. #kubeAPIServerOverride: "" + +# Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. +# https://golang.org/pkg/crypto/tls/#pkg-constants +# Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always +# prefer TLS1.3 Cipher Suites whenever possible. +#tlsCipherSuites: + +# TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +#tlsMinVersion: diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf index e856cda0be0..d6ec73e8bd4 100644 --- a/build/yamls/base/conf/antrea-controller.conf +++ b/build/yamls/base/conf/antrea-controller.conf @@ -27,3 +27,12 @@ featureGates: # And the Secret must be mounted to directory "/var/run/antrea/antrea-controller-tls" of the # antrea-controller container. #selfSignedCert: true + +# Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used. +# https://golang.org/pkg/crypto/tls/#pkg-constants +# Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always +# prefer TLS1.3 Cipher Suites whenever possible. +#tlsCipherSuites: + +# TLS min version from: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13. +#tlsMinVersion: diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 9fbb2b3ec64..680f7fd7920 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -51,6 +51,7 @@ import ( ofconfig "github.com/vmware-tanzu/antrea/pkg/ovs/openflow" "github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig" "github.com/vmware-tanzu/antrea/pkg/signals" + "github.com/vmware-tanzu/antrea/pkg/util/cipher" "github.com/vmware-tanzu/antrea/pkg/version" k8sproxy "github.com/vmware-tanzu/antrea/third_party/proxy" ) @@ -290,12 +291,18 @@ func run(o *Options) error { go proxier.Run(stopCh) } + cipherSuites, err := cipher.GenerateCipherSuitesList(o.config.TLSCipherSuites) + if err != nil { + return fmt.Errorf("error generating Cipher Suite list: %v", err) + } apiServer, err := apiserver.New( agentQuerier, networkPolicyController, o.config.APIPort, o.config.EnablePrometheusMetrics, - o.config.ClientConnection.Kubeconfig) + o.config.ClientConnection.Kubeconfig, + cipherSuites, + cipher.TLSVersionMap[o.config.TLSMinVersion]) if err != nil { return fmt.Errorf("error when creating agent API server: %v", err) } diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 68602961071..e2aabe0b558 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -127,4 +127,8 @@ type AgentConfig struct { // Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. // Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver. KubeAPIServerOverride string `yaml:"kubeAPIServerOverride,omitempty"` + // Cipher suites to use. + TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"` + // TLS min version. + TLSMinVersion string `yaml:"tlsMinVersion,omitempty"` } diff --git a/cmd/antrea-controller/config.go b/cmd/antrea-controller/config.go index a7bcee36d65..a58320f6689 100644 --- a/cmd/antrea-controller/config.go +++ b/cmd/antrea-controller/config.go @@ -39,4 +39,8 @@ type ControllerConfig struct { // antrea-controller container. // Defaults to true. SelfSignedCert bool `yaml:"selfSignedCert,omitempty"` + // Cipher suites to use. + TLSCipherSuites string `yaml:"tlsCipherSuites,omitempty"` + // TLS min version. + TLSMinVersion string `yaml:"tlsMinVersion,omitempty"` } diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 65f647e92b2..52b84a586a7 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -46,6 +46,7 @@ import ( "github.com/vmware-tanzu/antrea/pkg/log" "github.com/vmware-tanzu/antrea/pkg/monitor" "github.com/vmware-tanzu/antrea/pkg/signals" + "github.com/vmware-tanzu/antrea/pkg/util/cipher" "github.com/vmware-tanzu/antrea/pkg/version" ) @@ -145,6 +146,11 @@ func run(o *Options) error { statsAggregator = stats.NewAggregator(networkPolicyInformer, cnpInformer, anpInformer) } + cipherSuites, err := cipher.GenerateCipherSuitesList(o.config.TLSCipherSuites) + if err != nil { + return fmt.Errorf("error generating Cipher Suite list: %v", err) + } + apiServerConfig, err := createAPIServerConfig(o.config.ClientConnection.Kubeconfig, client, aggregatorClient, @@ -159,7 +165,9 @@ func run(o *Options) error { networkPolicyController, networkPolicyStatusController, statsAggregator, - o.config.EnablePrometheusMetrics) + o.config.EnablePrometheusMetrics, + cipherSuites, + cipher.TLSVersionMap[o.config.TLSMinVersion]) if err != nil { return fmt.Errorf("error creating API server config: %v", err) } @@ -224,7 +232,9 @@ func createAPIServerConfig(kubeconfig string, npController *networkpolicy.NetworkPolicyController, networkPolicyStatusController *networkpolicy.StatusController, statsAggregator *stats.Aggregator, - enableMetrics bool) (*apiserver.Config, error) { + enableMetrics bool, + cipherSuites []uint16, + tlsMinVersion uint16) (*apiserver.Config, error) { secureServing := genericoptions.NewSecureServingOptions().WithLoopback() authentication := genericoptions.NewDelegatingAuthenticationOptions() authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths(allowedPaths...) @@ -265,6 +275,8 @@ func createAPIServerConfig(kubeconfig string, serverConfig.OpenAPIConfig.Info.Title = "Antrea" serverConfig.EnableMetrics = enableMetrics serverConfig.MinRequestTimeout = int(serverMinWatchTimeout.Seconds()) + serverConfig.SecureServing.CipherSuites = cipherSuites + serverConfig.SecureServing.MinTLSVersion = tlsMinVersion return apiserver.NewConfig( serverConfig, diff --git a/pkg/agent/apiserver/apiserver.go b/pkg/agent/apiserver/apiserver.go index cd438fff756..9c96131243d 100644 --- a/pkg/agent/apiserver/apiserver.go +++ b/pkg/agent/apiserver/apiserver.go @@ -92,7 +92,7 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent // New creates an APIServer for running in antrea agent. func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, bindPort int, - enableMetrics bool, kubeconfig string) (*agentAPIServer, error) { + enableMetrics bool, kubeconfig string, cipherSuites []uint16, tlsMinVersion uint16) (*agentAPIServer, error) { cfg, err := newConfig(bindPort, enableMetrics, kubeconfig) if err != nil { return nil, err @@ -101,6 +101,8 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier if err != nil { return nil, err } + s.SecureServingInfo.CipherSuites = cipherSuites + s.SecureServingInfo.MinTLSVersion = tlsMinVersion if err := installAPIGroup(s, aq, npq); err != nil { return nil, err } diff --git a/pkg/util/cipher/cipher.go b/pkg/util/cipher/cipher.go new file mode 100644 index 00000000000..166fd2dc43a --- /dev/null +++ b/pkg/util/cipher/cipher.go @@ -0,0 +1,42 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cipher + +import ( + "crypto/tls" + "strings" + + "k8s.io/component-base/cli/flag" +) + +var TLSVersionMap = map[string]uint16{ + "VersionTLS10": tls.VersionTLS10, + "VersionTLS11": tls.VersionTLS11, + "VersionTLS12": tls.VersionTLS12, + "VersionTLS13": tls.VersionTLS13, +} + +// GenerateCipherSuitesList generates Cipher Suite list from comma-separated Cipher Suite string. +func GenerateCipherSuitesList(cipherSuites string) ([]uint16, error) { + csStrList := strings.Split(strings.ReplaceAll(cipherSuites, " ", ""), ",") + if len(csStrList) == 1 && csStrList[0] == "" { + return []uint16{}, nil + } + csIntList, err := flag.TLSCipherSuites(csStrList) + if err != nil { + return nil, err + } + return csIntList, nil +} diff --git a/pkg/util/cipher/cipher_test.go b/pkg/util/cipher/cipher_test.go new file mode 100644 index 00000000000..c949a25c5d9 --- /dev/null +++ b/pkg/util/cipher/cipher_test.go @@ -0,0 +1,52 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cipher + +import ( + "crypto/tls" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateCipherSuitesList(t *testing.T) { + cs0 := tls.TLS_RSA_WITH_RC4_128_SHA + cs1 := tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA + cs0Str := tls.CipherSuiteName(cs0) + cs1Str := tls.CipherSuiteName(cs1) + cs2StrNotExist := "TLS_RSA_WITH_3DES_EDE_CBC_SHA1234" + + tests := []struct { + str string + ids []uint16 + success bool + }{ + {fmt.Sprintf("%s,%s", cs0Str, cs1Str), []uint16{cs0, cs1}, true}, + {fmt.Sprintf(" %s, %s ", cs0Str, cs1Str), []uint16{cs0, cs1}, true}, + {fmt.Sprintf("%s,%s", cs0Str, cs2StrNotExist), []uint16{}, false}, + {" ", []uint16{}, true}, + } + + for _, tc := range tests { + output, err := GenerateCipherSuitesList(tc.str) + if tc.success { + assert.NoError(t, err) + assert.Equal(t, tc.ids, output) + } else { + assert.Error(t, err) + } + } +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 5eeb0519071..d6b9d714953 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -68,6 +68,7 @@ const ( antreaDefaultGW string = "antrea-gw0" testNamespace string = "antrea-test" busyboxContainerName string = "busybox" + controllerContainerName string = "antrea-controller" ovsContainerName string = "antrea-ovs" agentContainerName string = "antrea-agent" antreaYML string = "antrea.yml" diff --git a/test/e2e/tls_test.go b/test/e2e/tls_test.go new file mode 100644 index 00000000000..a26363d597d --- /dev/null +++ b/test/e2e/tls_test.go @@ -0,0 +1,151 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "crypto/tls" + "fmt" + "net" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/vmware-tanzu/antrea/pkg/apis" +) + +const ( + cipherSuite = tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 // a TLS1.2 Cipher Suite + cipherSuiteStr = "ECDHE-RSA-AES128-GCM-SHA256" +) + +var ( + cipherSuites = []uint16{cipherSuite} + opensslTLS13CipherSuites = []string{"TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", "TLS_CHACHA20_POLY1305_SHA256"} +) + +// TestAntreaApiserverTLSConfig tests Cipher Suite and TLSVersion config on Antrea apiserver, Controller side or Agent side. +func TestAntreaApiserverTLSConfig(t *testing.T) { + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + data.configureTLS(t, cipherSuites, "VersionTLS12") + + controllerPod, err := data.getAntreaController() + assert.NoError(t, err, "failed to get Antrea Controller Pod") + controllerPodName := controllerPod.Name + controlPlaneNode := controlPlaneNodeName() + agentPodName, err := data.getAntreaPodOnNode(controlPlaneNode) + assert.NoError(t, err, "failed to get Antrea Agent Pod Name on Control Plane Node") + + tests := []struct { + name string + podName string + containerName string + apiserver int + apiserverStr string + }{ + {"ControllerApiserver", controllerPodName, controllerContainerName, apis.AntreaControllerAPIPort, "Controller"}, + {"AgentApiserver", agentPodName, agentContainerName, apis.AntreaAgentAPIPort, "Agent"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data.checkTLS(t, tc.podName, tc.containerName, tc.apiserver, tc.apiserverStr) + }) + } +} + +func (data *TestData) configureTLS(t *testing.T, cipherSuites []uint16, tlsMinVersion string) { + var cipherSuitesStr string + for i, cs := range cipherSuites { + cipherSuitesStr = fmt.Sprintf("%s%s", cipherSuitesStr, tls.CipherSuiteName(cs)) + if i != len(cipherSuites)-1 { + cipherSuitesStr = fmt.Sprintf("%s,", cipherSuitesStr) + } + } + + if err := data.mutateAntreaConfigMap(func(data map[string]string) { + antreaControllerConf, _ := data["antrea-controller.conf"] + antreaControllerConf = strings.Replace(antreaControllerConf, "#tlsCipherSuites:", fmt.Sprintf("tlsCipherSuites: %s", cipherSuitesStr), 1) + antreaControllerConf = strings.Replace(antreaControllerConf, "#tlsMinVersion:", fmt.Sprintf("tlsMinVersion: %s", tlsMinVersion), 1) + data["antrea-controller.conf"] = antreaControllerConf + antreaAgentConf, _ := data["antrea-agent.conf"] + antreaAgentConf = strings.Replace(antreaAgentConf, "#tlsCipherSuites:", fmt.Sprintf("tlsCipherSuites: %s", cipherSuitesStr), 1) + antreaAgentConf = strings.Replace(antreaAgentConf, "#tlsMinVersion:", fmt.Sprintf("tlsMinVersion: %s", tlsMinVersion), 1) + data["antrea-agent.conf"] = antreaAgentConf + }, true, true); err != nil { + t.Fatalf("Failed to configure Cipher Suites and TLSMinVersion: %v", err) + } +} + +func (data *TestData) checkTLS(t *testing.T, podName string, containerName string, apiserver int, apiserverStr string) { + // 1. TLSMaxVersion unset, then a TLS1.3 Cipher Suite should be used. + stdouts := data.opensslConnect(t, podName, containerName, false, apiserver) + for _, stdout := range stdouts { + oneTLS13CS := false + for _, cs := range opensslTLS13CipherSuites { + if strings.Contains(stdout, fmt.Sprintf("New, TLSv1.3, Cipher is %s", cs)) { + oneTLS13CS = true + break + } + } + assert.True(t, oneTLS13CS, + "Cipher Suite used by %s apiserver should be a TLS1.3 one, output: %s", apiserverStr, stdout) + } + + // 2. Set TLSMaxVersion to TLS1.2, then TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 should be used + stdouts = data.opensslConnect(t, podName, containerName, true, apiserver) + for _, stdout := range stdouts { + assert.True(t, strings.Contains(stdout, fmt.Sprintf("New, TLSv1.2, Cipher is %s", cipherSuiteStr)), + "Cipher Suite used by %s apiserver should be the TLS1.2 one '%s', output: %s", apiserverStr, cipherSuiteStr, stdout) + } +} + +func (data *TestData) opensslConnect(t *testing.T, pod string, container string, tls12 bool, port int) []string { + var stdouts []string + opensslConnectCommands := []struct { + enabled bool + ip string + option string + }{ + { + clusterInfo.podV4NetworkCIDR != "", + "127.0.0.1", + "-4", + }, + { + clusterInfo.podV6NetworkCIDR != "", + "::1", + "-6", + }, + } + for _, c := range opensslConnectCommands { + if !c.enabled { + continue + } + cmd := []string{"timeout", "1", "openssl", "s_client", "-connect", net.JoinHostPort(c.ip, fmt.Sprint(port)), c.option} + if tls12 { + cmd = append(cmd, "-tls1_2") + } + stdout, stderr, err := data.runCommandFromPod(antreaNamespace, pod, container, cmd) + assert.NoError(t, err, "failed to run openssl command on Pod '%s'\nstderr: %s", pod, stderr) + t.Logf("Ran '%s' on Pod %s", strings.Join(cmd, " "), pod) + stdouts = append(stdouts, stdout) + } + return stdouts +}