diff --git a/examples/selfsigned-tls/selfsigned-ca.yaml b/examples/selfsigned-tls/selfsigned-ca.yaml new file mode 100644 index 0000000000..806a78be55 --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-ca.yaml @@ -0,0 +1,11 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: selfsigned-ca-cert +spec: + secretName: selfsigned-ca-cert + commonName: "certmanager" + isCA: true + issuerRef: + name: selfsigned-issuer + kind: Issuer diff --git a/examples/selfsigned-tls/selfsigned-cert-issuer.yaml b/examples/selfsigned-tls/selfsigned-cert-issuer.yaml new file mode 100644 index 0000000000..934b53124d --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-cert-issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-cert-issuer +spec: + ca: + secretName: selfsigned-ca-cert diff --git a/examples/selfsigned-tls/selfsigned-issuer.yaml b/examples/selfsigned-tls/selfsigned-issuer.yaml new file mode 100644 index 0000000000..7f06abf08a --- /dev/null +++ b/examples/selfsigned-tls/selfsigned-issuer.yaml @@ -0,0 +1,6 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} diff --git a/examples/selfsigned-tls/tidb-cluster.yaml b/examples/selfsigned-tls/tidb-cluster.yaml new file mode 100644 index 0000000000..aa93ea2274 --- /dev/null +++ b/examples/selfsigned-tls/tidb-cluster.yaml @@ -0,0 +1,28 @@ +apiVersion: pingcap.com/v1alpha1 +kind: TidbCluster +metadata: + name: tls +spec: + version: v3.0.8 + timezone: UTC + pvReclaimPolicy: Delete + pd: + baseImage: pingcap/pd + replicas: 1 + requests: + storage: "1Gi" + config: {} + tikv: + baseImage: pingcap/tikv + replicas: 1 + requests: + storage: "1Gi" + config: {} + tidb: + baseImage: pingcap/tidb + replicas: 1 + service: + type: ClusterIP + config: {} + tlsClient: + enabled: true diff --git a/examples/selfsigned-tls/tidb-server-cert.yaml b/examples/selfsigned-tls/tidb-server-cert.yaml new file mode 100644 index 0000000000..6580dc5091 --- /dev/null +++ b/examples/selfsigned-tls/tidb-server-cert.yaml @@ -0,0 +1,22 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: tidb-server-cert +spec: + secretName: tls-tidb-server-secret # -tidb-server-secret + subject: + organizationalUnits: + - "TiDB Operator" + organization: + - "PingCAP" + duration: "8760h" # 364 days + # If you want verify server cert Common Name (e.g. --ssl-verify-server-cert + # flag in MySQL CLI), you must configure the HostName you used to connect the + # server here. + commonName: "tls-tidb-server" + usages: + - "client auth" + - "server auth" + issuerRef: + name: selfsigned-cert-issuer + kind: Issuer diff --git a/hack/e2e-examples.sh b/hack/e2e-examples.sh index ee4355f29c..e1a5f6588f 100755 --- a/hack/e2e-examples.sh +++ b/hack/e2e-examples.sh @@ -33,12 +33,19 @@ hack/local-up-operator.sh echo "info: testing examples" export PATH=$PATH:$OUTPUT_BIN hack::ensure_kubectl -for t in $(find tests/examples/ -name '*.sh'); do + +cnt=0 +for t in $(find tests/examples/ -regextype sed -regex '.*/[0-9]\{3\}-.*\.sh'); do echo "info: testing $t" $t if [ $? -eq 0 ]; then echo "info: test $t passed" else echo "error: test $t failed" + $((cnt++)) fi done +if [ $cnt -gt 0 ]; then + echo "fatal: $cnt tests failed" + exit 1 +fi diff --git a/tests/examples/001-basic.sh b/tests/examples/001-basic.sh index 6e45aad216..8f4aec77a8 100755 --- a/tests/examples/001-basic.sh +++ b/tests/examples/001-basic.sh @@ -17,36 +17,20 @@ ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) cd $ROOT source "${ROOT}/hack/lib.sh" +source "${ROOT}/tests/examples/t.sh" + +NS=$(basename ${0%.*}) function cleanup() { - kubectl delete -f examples/basic/tidb-cluster.yaml + kubectl -n $NS delete -f examples/basic/tidb-cluster.yaml + kubectl delete ns $NS } trap cleanup EXIT -function checkReplicas() { - local pdDesiredReplicas="$1" - local tikvDesiredReplicas="$2" - local tidbDesiredReplicas="$3" - local pdReplicas=$(kubectl get tc basic -ojsonpath='{.status.pd.statefulSet.readyReplicas}') - if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then - echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas" - return 1 - fi - local tikvReplicas=$(kubectl get tc basic -ojsonpath='{.status.tikv.statefulSet.readyReplicas}') - if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then - echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas" - return 1 - fi - local tidbReplicas=$(kubectl get tc basic -ojsonpath='{.status.tidb.statefulSet.readyReplicas}') - if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then - echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas" - return 1 - fi - echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas" - return 0 -} +kubectl create ns $NS +hack::wait_for_success 10 3 "t::ns_is_active $NS" -kubectl apply -f examples/basic/tidb-cluster.yaml +kubectl -n $NS apply -f examples/basic/tidb-cluster.yaml -hack::wait_for_success 600 3 "checkReplicas 3 3 2" +hack::wait_for_success 600 3 "t::tc_is_ready $NS basic" diff --git a/tests/examples/002-selfsigned-tls.sh b/tests/examples/002-selfsigned-tls.sh new file mode 100755 index 0000000000..3816f81d3b --- /dev/null +++ b/tests/examples/002-selfsigned-tls.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) +cd $ROOT + +source "${ROOT}/hack/lib.sh" +source "${ROOT}/tests/examples/t.sh" + +NS=$(basename ${0%.*}) + +PORT_FORWARD_PID= + +function cleanup() { + if [ -n "$PORT_FORWARD_PID" ]; then + echo "info: kill port-forward background process (PID: $PORT_FORWARD_PID)" + kill $PORT_FORWARD_PID + fi + kubectl delete -f examples/selfsigned-tls/ --ignore-not-found + kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml --ignore-not-found + kubectl delete ns $NS +} + +trap cleanup EXIT + +kubectl create ns $NS +hack::wait_for_success 10 3 "t::ns_is_active $NS" + +kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml +hack::wait_for_success 10 3 "t::crds_are_ready certificaterequests.cert-manager.io certificates.cert-manager.io challenges.acme.cert-manager.io clusterissuers.cert-manager.io issuers.cert-manager.io orders.acme.cert-manager.io" +for d in cert-manager cert-manager-cainjector cert-manager-webhook; do + hack::wait_for_success 300 3 "t::deploy_is_ready cert-manager $d" + if [ $? -ne 0 ]; then + echo "fatal: timed out waiting for the deployment $d to be ready" + exit 1 + fi +done + +kubectl -n $NS apply -f examples/selfsigned-tls/ + +hack::wait_for_success 300 3 "t::tc_is_ready $NS tls" +if [ $? -ne 0 ]; then + echo "fatal: failed to wait for the cluster to be ready" + exit 1 +fi + +echo "info: verify mysql client can connect with tidb server with SSL enabled" +kubectl -n $NS port-forward svc/tls-tidb 4000:4000 &> /tmp/port-forward.log & +PORT_FORWARD_PID=$! + +host=127.0.0.1 +port=4000 +for ((i=0; i < 10; i++)); do + nc -zv -w 3 $host $port + if [ $? -eq 0 ]; then + break + else + echo "info: failed to connect to $host:$port, sleep 1 second then retry" + sleep 1 + fi +done + +hack::wait_for_success 100 3 "mysql -h 127.0.0.1 -P 4000 -uroot -e 'select tidb_version();'" +if [ $? -ne 0 ]; then + echo "fatal: failed to connect to TiDB" + exit 1 +fi + +has_ssl=$(mysql -h 127.0.0.1 -P 4000 -uroot --ssl -e "SHOW VARIABLES LIKE '%ssl%';" | awk '/have_ssl/ {print $2}') +if [[ "$has_ssl" != "YES" ]]; then + echo "fatal: ssl is not enabled successfully, has_ssl is '$has_ssl'" + exit 1 +fi +echo "info: ssl is enabled successfully, has_ssl is '$has_ssl'" diff --git a/tests/examples/t.sh b/tests/examples/t.sh new file mode 100644 index 0000000000..b25462f35b --- /dev/null +++ b/tests/examples/t.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Copyright 2020 PingCAP, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd) +cd $ROOT + +function t::tc_is_ready() { + local ns="$1" + local name="$2" + local pdDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.pd.replicas}') + local tikvDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tikv.replicas}') + local tidbDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tidb.replicas}') + local pdReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.pd.statefulSet.readyReplicas}') + if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then + echo "info: [tc/$name] got pd replicas $pdReplicas, expects $pdDesiredReplicas" + return 1 + fi + local tikvReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tikv.statefulSet.readyReplicas}') + if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then + echo "info: [tc/$name] got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas" + return 1 + fi + local tidbReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tidb.statefulSet.readyReplicas}') + if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then + echo "info: [tc/$name] got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas" + return 1 + fi + echo "info: [tc/$name] pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas" + return 0 +} + +function t::crds_are_ready() { + for name in $@; do + local established=$(kubectl get crd $name -o json | jq '.status["conditions"][] | select(.type == "Established") | .status') + if [ $? -ne 0 ]; then + echo "error: crd $name is not found" + return 1 + fi + if [[ "$established" != "True" ]]; then + echo "error: crd $name is not ready" + return 1 + fi + done + return 0 +} + +function t::ns_is_active() { + local ns="$1" + local phase=$(kubectl get ns $ns -ojsonpath='{.status.phase}') + [[ "$phase" == "Active" ]] +} + +function t::deploy_is_ready() { + local ns="$1" + local name="$2" + read a b <<<$(kubectl -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}') + if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then + echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)" + return 0 + fi + echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)" + return 1 +} +