use cert-manager to create and renew tidb-server certificates (self-signed example)

examples/selfsigned-tls/selfsigned-ca.yaml

@@ -0,0 +1,11 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: selfsigned-ca-cert
+spec:
+  secretName: selfsigned-ca-cert
+  commonName: "certmanager"
+  isCA: true
+  issuerRef:
+    name: selfsigned-issuer
+    kind: Issuer

examples/selfsigned-tls/selfsigned-cert-issuer.yaml

@@ -0,0 +1,7 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: selfsigned-cert-issuer
+spec:
+  ca:
+    secretName: selfsigned-ca-cert

examples/selfsigned-tls/selfsigned-issuer.yaml

@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: selfsigned-issuer
+spec:
+  selfSigned: {}

examples/selfsigned-tls/tidb-cluster.yaml

@@ -0,0 +1,28 @@
+apiVersion: pingcap.com/v1alpha1
+kind: TidbCluster
+metadata:
+  name: tls
+spec:
+  version: v3.0.8
+  timezone: UTC
+  pvReclaimPolicy: Delete
+  pd:
+    baseImage: pingcap/pd
+    replicas: 1
+    requests:
+      storage: "1Gi"
+    config: {}
+  tikv:
+    baseImage: pingcap/tikv
+    replicas: 1
+    requests:
+      storage: "1Gi"
+    config: {}
+  tidb:
+    baseImage: pingcap/tidb
+    replicas: 1
+    service:
+      type: ClusterIP
+    config: {}
+    tlsClient:
+      enabled: true 

examples/selfsigned-tls/tidb-server-cert.yaml

@@ -0,0 +1,22 @@
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: tidb-server-cert
+spec:
+  secretName: tls-tidb-server-secret # <cluster>-tidb-server-secret
+  subject:
+    organizationalUnits:
+    - "TiDB Operator"
+  organization:
+  - "PingCAP"
+  duration: "8760h" # 364 days
+  # If you want verify server cert Common Name (e.g. --ssl-verify-server-cert
+  # flag in MySQL CLI), you must configure the HostName you used to connect the
+  # server here.
+  commonName: "tls-tidb-server"
+  usages:
+  - "client auth"
+  - "server auth"
+  issuerRef:
+    name: selfsigned-cert-issuer
+    kind: Issuer

hack/e2e-examples.sh

@@ -33,12 +33,19 @@ hack/local-up-operator.sh
 echo "info: testing examples"
 export PATH=$PATH:$OUTPUT_BIN
 hack::ensure_kubectl
-for t in $(find tests/examples/ -name '*.sh'); do
+
+cnt=0
+for t in $(find tests/examples/ -regextype sed -regex '.*/[0-9]\{3\}-.*\.sh'); do
     echo "info: testing $t"
     $t
     if [ $? -eq 0 ]; then
         echo "info: test $t passed"
     else
         echo "error: test $t failed"
+        $((cnt++))
     fi
 done
+if [ $cnt -gt 0 ]; then
+    echo "fatal: $cnt tests failed"
+    exit 1
+fi

tests/examples/001-basic.sh

@@ -17,36 +17,20 @@ ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
 cd $ROOT
 
 source "${ROOT}/hack/lib.sh"
+source "${ROOT}/tests/examples/t.sh"
+
+NS=$(basename ${0%.*})
 
 function cleanup() {
-    kubectl delete -f examples/basic/tidb-cluster.yaml 
+    kubectl -n $NS delete -f examples/basic/tidb-cluster.yaml
+    kubectl delete ns $NS
 }
 
 trap cleanup EXIT
 
-function checkReplicas() {
-    local pdDesiredReplicas="$1"
-    local tikvDesiredReplicas="$2"
-    local tidbDesiredReplicas="$3"
-    local pdReplicas=$(kubectl get tc basic -ojsonpath='{.status.pd.statefulSet.readyReplicas}')
-    if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then
-        echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas"
-        return 1
-    fi
-    local tikvReplicas=$(kubectl get tc basic -ojsonpath='{.status.tikv.statefulSet.readyReplicas}')
-    if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then
-        echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas"
-        return 1
-    fi
-    local tidbReplicas=$(kubectl get tc basic -ojsonpath='{.status.tidb.statefulSet.readyReplicas}')
-    if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then
-        echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas"
-        return 1
-    fi
-    echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas"
-    return 0
-}
+kubectl create ns $NS
+hack::wait_for_success 10 3 "t::ns_is_active $NS"
 
-kubectl apply -f examples/basic/tidb-cluster.yaml 
+kubectl -n $NS apply -f examples/basic/tidb-cluster.yaml
 
-hack::wait_for_success 600 3 "checkReplicas 3 3 2"
+hack::wait_for_success 600 3 "t::tc_is_ready $NS basic"

tests/examples/002-selfsigned-tls.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
+cd $ROOT
+
+source "${ROOT}/hack/lib.sh"
+source "${ROOT}/tests/examples/t.sh"
+
+NS=$(basename ${0%.*})
+
+PORT_FORWARD_PID=
+
+function cleanup() {
+    if [ -n "$PORT_FORWARD_PID" ]; then
+        echo "info: kill port-forward background process (PID: $PORT_FORWARD_PID)"
+        kill $PORT_FORWARD_PID
+    fi
+    kubectl delete -f examples/selfsigned-tls/ --ignore-not-found
+    kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml --ignore-not-found
+    kubectl delete ns $NS
+}
+
+trap cleanup EXIT
+
+kubectl create ns $NS
+hack::wait_for_success 10 3 "t::ns_is_active $NS"
+
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml
+hack::wait_for_success 10 3 "t::crds_are_ready certificaterequests.cert-manager.io certificates.cert-manager.io challenges.acme.cert-manager.io clusterissuers.cert-manager.io issuers.cert-manager.io orders.acme.cert-manager.io"
+for d in cert-manager cert-manager-cainjector cert-manager-webhook; do
+    hack::wait_for_success 300 3 "t::deploy_is_ready cert-manager $d"
+    if [ $? -ne 0 ]; then
+        echo "fatal: timed out waiting for the deployment $d to be ready"
+        exit 1
+    fi
+done
+
+kubectl -n $NS apply -f examples/selfsigned-tls/
+
+hack::wait_for_success 300 3 "t::tc_is_ready $NS tls"
+if [ $? -ne 0 ]; then
+    echo "fatal: failed to wait for the cluster to be ready"
+    exit 1
+fi
+
+echo "info: verify mysql client can connect with tidb server with SSL enabled"
+kubectl -n $NS port-forward svc/tls-tidb 4000:4000 &> /tmp/port-forward.log &
+PORT_FORWARD_PID=$!
+
+host=127.0.0.1
+port=4000
+for ((i=0; i < 10; i++)); do
+	nc -zv -w 3 $host $port
+	if [ $? -eq 0 ]; then
+		break
+	else
+		echo "info: failed to connect to $host:$port, sleep 1 second then retry"
+		sleep 1
+	fi
+done
+
+hack::wait_for_success 100 3 "mysql -h 127.0.0.1 -P 4000 -uroot -e 'select tidb_version();'"
+if [ $? -ne 0 ]; then
+    echo "fatal: failed to connect to TiDB"
+    exit 1
+fi
+
+has_ssl=$(mysql -h 127.0.0.1 -P 4000 -uroot --ssl -e "SHOW VARIABLES LIKE '%ssl%';" | awk '/have_ssl/ {print $2}')
+if [[ "$has_ssl" != "YES" ]]; then
+	echo "fatal: ssl is not enabled successfully, has_ssl is '$has_ssl'"
+	exit 1
+fi
+echo "info: ssl is enabled successfully, has_ssl is '$has_ssl'"

tests/examples/t.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
+cd $ROOT
+
+function t::tc_is_ready() {
+    local ns="$1"
+    local name="$2"
+    local pdDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.pd.replicas}')
+    local tikvDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tikv.replicas}')
+    local tidbDesiredReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.spec.tidb.replicas}')
+    local pdReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.pd.statefulSet.readyReplicas}')
+    if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then
+        echo "info: [tc/$name] got pd replicas $pdReplicas, expects $pdDesiredReplicas"
+        return 1
+    fi
+    local tikvReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tikv.statefulSet.readyReplicas}')
+    if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then
+        echo "info: [tc/$name] got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas"
+        return 1
+    fi
+    local tidbReplicas=$(kubectl -n $ns get tc $name -ojsonpath='{.status.tidb.statefulSet.readyReplicas}')
+    if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then
+        echo "info: [tc/$name] got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas"
+        return 1
+    fi
+    echo "info: [tc/$name] pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas"
+    return 0
+}
+
+function t::crds_are_ready() {
+    for name in $@; do
+        local established=$(kubectl get crd $name -o json | jq '.status["conditions"][] | select(.type == "Established") | .status')
+        if [ $? -ne 0 ]; then
+            echo "error: crd $name is not found"
+            return 1
+        fi
+        if [[ "$established" != "True" ]]; then
+            echo "error: crd $name is not ready"
+            return 1
+        fi
+    done
+    return 0
+}
+
+function t::ns_is_active() {
+    local ns="$1"
+    local phase=$(kubectl get ns $ns -ojsonpath='{.status.phase}')
+    [[ "$phase" == "Active" ]]
+}
+
+function t::deploy_is_ready() {
+    local ns="$1"
+    local name="$2"
+    read a b <<<$(kubectl -n $ns get deploy/$name -ojsonpath='{.spec.replicas} {.status.readyReplicas}{"\n"}')
+    if [[ "$a" -gt 0 && "$a" -eq "$b" ]]; then
+        echo "info: all pods of deployment $ns/$name are ready (desired: $a, ready: $b)"
+        return 0
+    fi
+    echo "info: pods of deployment $ns/$name (desired: $a, ready: $b)"
+    return 1
+}
+