From 06a87eeb578639c962d6e12aa41f143eee09c83d Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Mon, 2 Mar 2020 19:18:28 +0800 Subject: [PATCH 1/2] selfsigned tls cert created by cert-manager --- examples/selfsigned-tls/selfsigned-ca.yaml | 11 +++++++ .../selfsigned-cert-issuer.yaml | 7 +++++ .../selfsigned-tls/selfsigned-issuer.yaml | 6 ++++ examples/selfsigned-tls/tidb-client-cert.yaml | 30 +++++++++++++++++++ examples/selfsigned-tls/tidb-cluster.yaml | 29 ++++++++++++++++++ 5 files changed, 83 insertions(+) create mode 100644 examples/selfsigned-tls/selfsigned-ca.yaml create mode 100644 examples/selfsigned-tls/selfsigned-cert-issuer.yaml create mode 100644 examples/selfsigned-tls/selfsigned-issuer.yaml create mode 100644 examples/selfsigned-tls/tidb-client-cert.yaml create mode 100644 examples/selfsigned-tls/tidb-cluster.yaml diff --git a/examples/selfsigned-tls/selfsigned-ca.yaml b/examples/selfsigned-tls/selfsigned-ca.yaml new file mode 100644 index 0000000000..806a78be55 --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-ca.yaml @@ -0,0 +1,11 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: selfsigned-ca-cert +spec: + secretName: selfsigned-ca-cert + commonName: "certmanager" + isCA: true + issuerRef: + name: selfsigned-issuer + kind: Issuer diff --git a/examples/selfsigned-tls/selfsigned-cert-issuer.yaml b/examples/selfsigned-tls/selfsigned-cert-issuer.yaml new file mode 100644 index 0000000000..934b53124d --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-cert-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-cert-issuer +spec: + ca: + secretName: selfsigned-ca-cert diff --git a/examples/selfsigned-tls/selfsigned-issuer.yaml b/examples/selfsigned-tls/selfsigned-issuer.yaml new file mode 100644 index 0000000000..7f06abf08a --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-issuer.yaml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} diff --git a/examples/selfsigned-tls/tidb-client-cert.yaml b/examples/selfsigned-tls/tidb-client-cert.yaml new file mode 100644 index 0000000000..b698c3011e --- /dev/null +++ b/examples/selfsigned-tls/tidb-client-cert.yaml @@ -0,0 +1,30 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: tidb-client-cert +spec: + secretName: tidb-client-cert + subject: + organizationalUnits: + - "TiDB Operator" + organization: + - "PingCAP" + duration: "8760h" # 364 days + commonName: "basic-tidb" + dnsNames: + - basic-tidb.default + - basic-tidb.default.svc + - basic-tidb-peer.default + - basic-tidb-peer.default.svc + - "*.basic-tidb-peer.default" + - "*.basic-tidb-peer.default.svc" + - "localhost" + ipAddresses: + - "127.0.0.1" + - "::1" + usages: + - "client auth" + - "server auth" + issuerRef: + name: selfsigned-cert-issuer + kind: Issuer diff --git a/examples/selfsigned-tls/tidb-cluster.yaml b/examples/selfsigned-tls/tidb-cluster.yaml new file mode 100644 index 0000000000..43f78fe218 --- /dev/null +++ b/examples/selfsigned-tls/tidb-cluster.yaml @@ -0,0 +1,29 @@ +apiVersion: pingcap.com/v1alpha1 +kind: TidbCluster +metadata: + name: basic +spec: + version: v3.0.8 + timezone: UTC + pvReclaimPolicy: Delete + pd: + baseImage: pingcap/pd + replicas: 3 + requests: + storage: "1Gi" + config: {} + tikv: + baseImage: pingcap/tikv + replicas: 3 + requests: + storage: "1Gi" + config: {} + tidb: + baseImage: pingcap/tidb + replicas: 2 + service: + type: ClusterIP + config: {} + tlsClient: + enabled: true + secretName: tidb-client-cert From 0e254d48600cb91194f4c319e3ae3e5419d02b3c Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Tue, 10 Mar 2020 21:49:49 +0800 Subject: [PATCH 2/2] add tests --- examples/selfsigned-tls/tidb-client-cert.yaml | 30 ------- examples/selfsigned-tls/tidb-cluster.yaml | 9 +- examples/selfsigned-tls/tidb-server-cert.yaml | 22 +++++ hack/e2e-examples.sh | 9 +- tests/examples/001-basic.sh | 34 ++------ tests/examples/002-selfsigned-tls.sh | 86 +++++++++++++++++++ tests/examples/t.sh | 76 ++++++++++++++++ 7 files changed, 205 insertions(+), 61 deletions(-) delete mode 100644 examples/selfsigned-tls/tidb-client-cert.yaml create mode 100644 examples/selfsigned-tls/tidb-server-cert.yaml create mode 100755 tests/examples/002-selfsigned-tls.sh create mode 100644 tests/examples/t.sh diff --git a/examples/selfsigned-tls/tidb-client-cert.yaml b/examples/selfsigned-tls/tidb-client-cert.yaml deleted file mode 100644 index b698c3011e..0000000000 --- a/examples/selfsigned-tls/tidb-client-cert.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: cert-manager.io/v1alpha2 -kind: Certificate -metadata: - name: tidb-client-cert -spec: - secretName: tidb-client-cert - subject: - organizationalUnits: - - "TiDB Operator" - organization: - - "PingCAP" - duration: "8760h" # 364 days - commonName: "basic-tidb" - dnsNames: - - basic-tidb.default - - basic-tidb.default.svc - - basic-tidb-peer.default - - basic-tidb-peer.default.svc - - "*.basic-tidb-peer.default" - - "*.basic-tidb-peer.default.svc" - - "localhost" - ipAddresses: - - "127.0.0.1" - - "::1" - usages: - - "client auth" - - "server auth" - issuerRef: - name: selfsigned-cert-issuer - kind: Issuer diff --git a/examples/selfsigned-tls/tidb-cluster.yaml b/examples/selfsigned-tls/tidb-cluster.yaml index 43f78fe218..aa93ea2274 100644 --- a/examples/selfsigned-tls/tidb-cluster.yaml +++ b/examples/selfsigned-tls/tidb-cluster.yaml @@ -1,29 +1,28 @@ apiVersion: pingcap.com/v1alpha1 kind: TidbCluster metadata: - name: basic + name: tls spec: version: v3.0.8 timezone: UTC pvReclaimPolicy: Delete pd: baseImage: pingcap/pd - replicas: 3 + replicas: 1 requests: storage: "1Gi" config: {} tikv: baseImage: pingcap/tikv - replicas: 3 + replicas: 1 requests: storage: "1Gi" config: {} tidb: baseImage: pingcap/tidb - replicas: 2 + replicas: 1 service: type: ClusterIP config: {} tlsClient: enabled: true - secretName: tidb-client-cert diff --git a/examples/selfsigned-tls/tidb-server-cert.yaml b/examples/selfsigned-tls/tidb-server-cert.yaml new file mode 100644 index 0000000000..6580dc5091 --- /dev/null +++ b/examples/selfsigned-tls/tidb-server-cert.yaml @@ -0,0 +1,22 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: tidb-server-cert +spec: + secretName: tls-tidb-server-secret # -tidb-server-secret + subject: + organizationalUnits: + - "TiDB Operator" + organization: + - "PingCAP" + duration: "8760h" # 364 days + # If you want verify server cert Common Name (e.g. --ssl-verify-server-cert + # flag in MySQL CLI), you must configure the HostName you used to connect the + # server here. + commonName: "tls-tidb-server" + usages: + - "client auth" + - "server auth" + issuerRef: + name: selfsigned-cert-issuer + kind: Issuer diff --git a/hack/e2e-examples.sh b/hack/e2e-examples.sh index ee4355f29c..e1a5f6588f 100755 --- a/hack/e2e-examples.sh +++ b/hack/e2e-examples.sh @@ -33,12 +33,19 @@ hack/local-up-operator.sh echo "info: testing examples" export PATH=$PATH:$OUTPUT_BIN hack::ensure_kubectl -for t in $(find tests/examples/ -name '*.sh'); do + +cnt=0 +for t in $(find tests/examples/ -regextype sed -regex '.*/[0-9]\{3\}-.*\.sh'); do echo "info: testing $t" $t if [ $? -eq 0 ]; then echo "info: test $t passed" else echo "error: test $t failed" + $((cnt++)) fi done +if [ $cnt -gt 0 ]; then + echo "fatal: $cnt tests failed" + exit 1 +fi diff --git a/tests/examples/001-basic.sh b/tests/examples/001-basic.sh index 6e45aad216..8f4aec77a8 100755 --- a/tests/examples/001-basic.sh +++ b/tests/examples/001-basic.sh @@ -17,36 +17,20 @@ ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd $ROOT source "${ROOT}/hack/lib.sh" +source "${ROOT}/tests/examples/t.sh" + +NS=$(basename ${0%.*}) function cleanup() { - kubectl delete -f examples/basic/tidb-cluster.yaml + kubectl -n $NS delete -f examples/basic/tidb-cluster.yaml + kubectl delete ns $NS } trap cleanup EXIT -function checkReplicas() { - local pdDesiredReplicas="$1" - local tikvDesiredReplicas="$2" - local tidbDesiredReplicas="$3" - local pdReplicas=$(kubectl get tc basic -ojsonpath='{.status.pd.statefulSet.readyReplicas}') - if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then - echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas" - return 1 - fi - local tikvReplicas=$(kubectl get tc basic -ojsonpath='{.status.tikv.statefulSet.readyReplicas}') - if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then - echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas" - return 1 - fi - local tidbReplicas=$(kubectl get tc basic -ojsonpath='{.status.tidb.statefulSet.readyReplicas}') - if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then - echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas" - return 1 - fi - echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas" - return 0 -} +kubectl create ns $NS +hack::wait_for_success 10 3 "t::ns_is_active $NS" -kubectl apply -f examples/basic/tidb-cluster.yaml +kubectl -n $NS apply -f examples/basic/tidb-cluster.yaml -hack::wait_for_success 600 3 "checkReplicas 3 3 2" +hack::wait_for_success 600 3 "t::tc_is_ready $NS basic" diff --git a/tests/examples/002-selfsigned-tls.sh b/tests/examples/002-selfsigned-tls.sh new file mode 100755 index 0000000000..3816f81d3b --- /dev/null +++ b/tests/examples/002-selfsigned-tls.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) +cd $ROOT + +source "${ROOT}/hack/lib.sh" +source "${ROOT}/tests/examples/t.sh" + +NS=$(basename ${0%.*}) + +PORT_FORWARD_PID= + +function cleanup() { + if [ -n "$PORT_FORWARD_PID" ]; then + echo "info: kill port-forward background process (PID: $PORT_FORWARD_PID)" + kill $PORT_FORWARD_PID + fi + kubectl delete -f examples/selfsigned-tls/ --ignore-not-found + kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml --ignore-not-found + kubectl delete ns $NS +} + +trap cleanup EXIT + +kubectl create ns $NS +hack::wait_for_success 10 3 "t::ns_is_active $NS" + +kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml +hack::wait_for_success 10 3 "t::crds_are_ready certificaterequests.cert-manager.io certificates.cert-manager.io challenges.acme.cert-manager.io clusterissuers.cert-manager.io issuers.cert-manager.io orders.acme.cert-manager.io" +for d in cert-manager cert-manager-cainjector cert-manager-webhook; do + hack::wait_for_success 300 3 "t::deploy_is_ready cert-manager $d" + if [ $? -ne 0 ]; then + echo "fatal: timed out waiting for the deployment $d to be ready" + exit 1 + fi +done + +kubectl -n $NS apply -f examples/selfsigned-tls/ + +hack::wait_for_success 300 3 "t::tc_is_ready $NS tls" +if [ $? -ne 0 ]; then + echo "fatal: failed to wait for the cluster to be ready" + exit 1 +fi + +echo "info: verify mysql client can connect with tidb server with SSL enabled" +kubectl -n $NS port-forward svc/tls-tidb 4000:4000 &> /tmp/port-forward.log & +PORT_FORWARD_PID=$! + +host=127.0.0.1 +port=4000 +for ((i=0; i < 10; i++)); do + nc -zv -w 3 $host $port + if [ $? -eq 0 ]; then + break + else + echo "info: failed to connect to $host:$port, sleep 1 second then retry" + sleep 1 + fi +done + +hack::wait_for_success 100 3 "mysql -h 127.0.0.1 -P 4000 -uroot -e 'select tidb_version();'" +if [ $? -ne 0 ]; then + echo "fatal: failed to connect to TiDB" + exit 1 +fi + +has_ssl=$(mysql -h 127.0.0.1 -P 4000 -uroot --ssl -e "SHOW VARIABLES LIKE '%ssl%';" | awk '/have_ssl/ {print $2}') +if [[ "$has_ssl" != "YES" ]]; then + echo "fatal: ssl is not enabled successfully, has_ssl is '$has_ssl'" + exit 1 +fi +echo "info: ssl is enabled successfully, has_ssl is '$has_ssl'" diff --git a/tests/examples/t.sh b/tests/examples/t.sh new file mode 100644 index 0000000000..b25462f35b --- /dev/null +++ b/tests/examples/t.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) +cd $ROOT + +function t::tc_is_ready() { + local ns="$1" + local name="$2" + local pdDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.pd.replicas}') + local tikvDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tikv.replicas}') + local tidbDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tidb.replicas}') + local pdReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.pd.statefulSet.readyReplicas}') + if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then + echo "info: [tc/$name] got pd replicas $pdReplicas, expects $pdDesiredReplicas" + return 1 + fi + local tikvReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tikv.statefulSet.readyReplicas}') + if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then + echo "info: [tc/$name] got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas" + return 1 + fi + local tidbReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tidb.statefulSet.readyReplicas}') + if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then + echo "info: [tc/$name] got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas" + return 1 + fi + echo "info: [tc/$name] pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas" + return 0 +} + +function t::crds_are_ready() { + for name in $@; do + local established=$(kubectl get crd $name -o json | jq '.status["conditions"][] | select(.type == "Established") | .status') + if [ $? -ne 0 ]; then + echo "error: crd $name is not found" + return 1 + fi + if [[ "$established" != "True" ]]; then + echo "error: crd $name is not ready" + return 1 + fi + done + return 0 +} + +function t::ns_is_active() { + local ns="$1" + local phase=$(kubectl get ns $ns -ojsonpath='{.status.phase}') + [[ "$phase" == "Active" ]] +} + +function t::deploy_is_ready() { + local ns="$1" + local name="$2" + read a b <<<$(kubectl -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}') + if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then + echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)" + return 0 + fi + echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)" + return 1 +} +