From 730edbe13680b2130195ebab32004679cac168ee Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Tue, 10 Mar 2020 21:49:49 +0800
Subject: [PATCH] add tests

---
 examples/selfsigned-tls/tidb-client-cert.yaml | 18 +++---
 examples/selfsigned-tls/tidb-cluster.yaml     |  3 +-
 hack/e2e-examples.sh                          |  4 +-
 tests/examples/001-basic.sh                   | 26 +--------
 tests/examples/002-selfsigned-tls.sh          | 56 ++++++++++++++++++
 tests/examples/t.sh                           | 57 +++++++++++++++++++
 6 files changed, 127 insertions(+), 37 deletions(-)
 create mode 100755 tests/examples/002-selfsigned-tls.sh
 create mode 100644 tests/examples/t.sh

diff --git a/examples/selfsigned-tls/tidb-client-cert.yaml b/examples/selfsigned-tls/tidb-client-cert.yaml
index b698c3011e5..d62273f6168 100644
--- a/examples/selfsigned-tls/tidb-client-cert.yaml
+++ b/examples/selfsigned-tls/tidb-client-cert.yaml
@@ -1,23 +1,23 @@
 apiVersion: cert-manager.io/v1alpha2
 kind: Certificate
 metadata:
-  name: tidb-client-cert
+  name: tidb-server-cert
 spec:
-  secretName: tidb-client-cert
+  secretName: tls-tidb-server-secret # <cluster>-tidb-server-secret
   subject:
     organizationalUnits:
     - "TiDB Operator"
   organization:
   - "PingCAP"
   duration: "8760h" # 364 days
-  commonName: "basic-tidb"
+  commonName: "tls-tidb"
   dnsNames:
-  - basic-tidb.default
-  - basic-tidb.default.svc
-  - basic-tidb-peer.default
-  - basic-tidb-peer.default.svc
-  - "*.basic-tidb-peer.default"
-  - "*.basic-tidb-peer.default.svc"
+  - tls-tidb.default
+  - tls-tidb.default.svc
+  - tls-tidb-peer.default
+  - tls-tidb-peer.default.svc
+  - "*.tls-tidb-peer.default"
+  - "*.tls-tidb-peer.default.svc"
   - "localhost"
   ipAddresses:
   - "127.0.0.1"
diff --git a/examples/selfsigned-tls/tidb-cluster.yaml b/examples/selfsigned-tls/tidb-cluster.yaml
index 43f78fe2181..d7fb5ce35c6 100644
--- a/examples/selfsigned-tls/tidb-cluster.yaml
+++ b/examples/selfsigned-tls/tidb-cluster.yaml
@@ -1,7 +1,7 @@
 apiVersion: pingcap.com/v1alpha1
 kind: TidbCluster
 metadata:
-  name: basic
+  name: tls
 spec:
   version: v3.0.8
   timezone: UTC
@@ -26,4 +26,3 @@ spec:
     config: {}
     tlsClient:
       enabled: true 
-      secretName: tidb-client-cert
diff --git a/hack/e2e-examples.sh b/hack/e2e-examples.sh
index ee4355f29c2..06798370209 100755
--- a/hack/e2e-examples.sh
+++ b/hack/e2e-examples.sh
@@ -33,9 +33,9 @@ hack/local-up-operator.sh
 echo "info: testing examples"
 export PATH=$PATH:$OUTPUT_BIN
 hack::ensure_kubectl
-for t in $(find tests/examples/ -name '*.sh'); do
+for t in $(find tests/examples/ -regextype sed -regex '.*/[0-9]\{3\}-.*\.sh'); do
     echo "info: testing $t"
-    $t
+    echo $t
     if [ $? -eq 0 ]; then
         echo "info: test $t passed"
     else
diff --git a/tests/examples/001-basic.sh b/tests/examples/001-basic.sh
index 6e45aad2163..54ad62a3da8 100755
--- a/tests/examples/001-basic.sh
+++ b/tests/examples/001-basic.sh
@@ -17,6 +17,7 @@ ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
 cd $ROOT
 
 source "${ROOT}/hack/lib.sh"
+source "${ROOT}/tests/examples/t.sh"
 
 function cleanup() {
     kubectl delete -f examples/basic/tidb-cluster.yaml 
@@ -24,29 +25,6 @@ function cleanup() {
 
 trap cleanup EXIT
 
-function checkReplicas() {
-    local pdDesiredReplicas="$1"
-    local tikvDesiredReplicas="$2"
-    local tidbDesiredReplicas="$3"
-    local pdReplicas=$(kubectl get tc basic -ojsonpath='{.status.pd.statefulSet.readyReplicas}')
-    if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then
-        echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas"
-        return 1
-    fi
-    local tikvReplicas=$(kubectl get tc basic -ojsonpath='{.status.tikv.statefulSet.readyReplicas}')
-    if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then
-        echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas"
-        return 1
-    fi
-    local tidbReplicas=$(kubectl get tc basic -ojsonpath='{.status.tidb.statefulSet.readyReplicas}')
-    if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then
-        echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas"
-        return 1
-    fi
-    echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas"
-    return 0
-}
-
 kubectl apply -f examples/basic/tidb-cluster.yaml 
 
-hack::wait_for_success 600 3 "checkReplicas 3 3 2"
+hack::wait_for_success 600 3 "t::tc_is_ready default basic"
diff --git a/tests/examples/002-selfsigned-tls.sh b/tests/examples/002-selfsigned-tls.sh
new file mode 100755
index 00000000000..80000a5096a
--- /dev/null
+++ b/tests/examples/002-selfsigned-tls.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
+cd $ROOT
+
+source "${ROOT}/hack/lib.sh"
+source "${ROOT}/tests/examples/t.sh"
+
+function cleanup() {
+    kubectl delete -f examples/selfsigned-tls/ --ignore-not-found
+    kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml --ignore-not-found
+}
+
+trap cleanup EXIT
+
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v0.13.1/cert-manager.yaml
+hack::wait_for_success 10 3 "t::crds_are_ready certificaterequests.cert-manager.io certificates.cert-manager.io challenges.acme.cert-manager.io clusterissuers.cert-manager.io issuers.cert-manager.io orders.acme.cert-manager.io"
+
+kubectl apply -f examples/selfsigned-tls/
+
+hack::wait_for_success 600 3 "t::tc_is_ready default tls"
+
+echo "info: verify mysql client can connect with tidb server with SSL enabled"
+kubectl port-forward svc/tls-tidb 4000:4000 &> /tmp/port-forward.log &
+
+host=127.0.0.1
+port=4000
+for ((i=0; i < 10; i++)); do
+	nc -zv -w 3 $host $port
+	if [ $? -eq 0 ]; then
+		break
+	else
+		echo "info: failed to connect to $host:$port, sleep 1 second then retry"
+		sleep 1
+	fi
+done
+
+hack::wait_for_success 100 3 "mysql -h 127.0.0.1 -P 4000 -uroot -e 'select tidb_version();'"
+has_ssl=$(mysql -h 127.0.0.1 -P 4000 -uroot --ssl -e "SHOW VARIABLES LIKE '%ssl%';" | awk '/have_ssl/ {print $2}')
+if [[ "$has_ssl" != "Yes" ]]; then
+	echo "error: ssl is not enabled successfully, got '$has_ssl'"
+	exit 1
+fi
diff --git a/tests/examples/t.sh b/tests/examples/t.sh
new file mode 100644
index 00000000000..0b55b880728
--- /dev/null
+++ b/tests/examples/t.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Copyright 2020 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
+cd $ROOT
+
+function t::tc_is_ready() {
+    local ns="$1"
+    local name="$2"
+    local pdDesiredReplicas=$(kubectl get tc $name -ojsonpath='{.spec.pd.replicas}')
+    local tikvDesiredReplicas=$(kubectl get tc $name -ojsonpath='{.spec.tikv.replicas}')
+    local tidbDesiredReplicas=$(kubectl get tc $name -ojsonpath='{.spec.tidb.replicas}')
+    local pdReplicas=$(kubectl get tc $name -ojsonpath='{.status.pd.statefulSet.readyReplicas}')
+    if [[ "$pdReplicas" != "$pdDesiredReplicas" ]]; then
+        echo "info: got pd replicas $pdReplicas, expects $pdDesiredReplicas"
+        return 1
+    fi
+    local tikvReplicas=$(kubectl get tc $name -ojsonpath='{.status.tikv.statefulSet.readyReplicas}')
+    if [[ "$tikvReplicas" != "$tikvDesiredReplicas" ]]; then
+        echo "info: got tikv replicas $tikvReplicas, expects $tikvDesiredReplicas"
+        return 1
+    fi
+    local tidbReplicas=$(kubectl get tc $name -ojsonpath='{.status.tidb.statefulSet.readyReplicas}')
+    if [[ "$tidbReplicas" != "$tidbDesiredReplicas" ]]; then
+        echo "info: got tidb replicas $tidbReplicas, expects $tidbDesiredReplicas"
+        return 1
+    fi
+    echo "info: pd replicas $pdReplicas, tikv replicas $tikvReplicas, tidb replicas $tidbReplicas"
+    return 0
+}
+
+function t::crds_are_ready() {
+    for name in $@; do
+        local established=$(kubectl get crd $name -o json | jq '.status["conditions"][] | select(.type == "Established") | .status')
+        if [ $? -ne 0 ]; then
+            echo "error: $name is not found"
+            return 1
+        fi
+        if [[ "$established" != "True" ]]; then
+            echo "error: $name is not ready"
+            return 1
+        fi
+    done
+    return 0
+}