fix: probes command

README.md

@@ -34,7 +34,7 @@ See the [Resources and Labels](./doc/user/resource_labels.md) doc for an overvie
 ## Requirements
 
 - Kubernetes 1.8+
-- etcd 3.2.13+
+- etcd 3.3+
 
 ## Demo
 
@@ -126,7 +126,7 @@ metadata:
   name: "example-etcd-cluster"
 spec:
   size: 3
-  version: "v3.2.13"
+  version: "v3.4.19"
 ```
 ```
 $ kubectl apply -f example/example-etcd-cluster.yaml

example/example-etcd-cluster.yaml

@@ -8,4 +8,4 @@ metadata:
   #   etcd.database.coreos.com/scope: clusterwide
 spec:
   size: 3
-  version: "v3.2.13"
+  version: "v3.4.19"

pkg/apis/etcd/v1beta2/cluster.go

@@ -24,7 +24,7 @@ import (
 
 const (
 	defaultRepository  = "quay.io/coreos/etcd"
-	DefaultEtcdVersion = "v3.2.13"
+	DefaultEtcdVersion = "v3.4.19"
 )
 
 var (
@@ -83,10 +83,10 @@ type ClusterSpec struct {
 	// The etcd-operator will eventually make the etcd cluster version
 	// equal to the expected version.
 	//
-	// The version must follow the [semver]( http://semver.org) format, for example "3.2.13".
+	// The version must follow the [semver]( http://semver.org) format, for example "v3.4.19".
 	// Only etcd released versions are supported: https://github.com/coreos/etcd/releases
 	//
-	// If version is not set, default is "v3.2.13".
+	// If version is not set, default is "v3.4.19".
 	Version string `json:"version,omitempty"`
 
 	// Paused is to pause the control of the operator for the etcd cluster.

pkg/util/k8sutil/k8sutil.go

@@ -423,12 +423,51 @@ func setupClientServiceURL(endpoint string) string {
 	return fmt.Sprintf("http://%s:2379", endpoint)
 }
 
+func setupInitContainerCommand(cs api.ClusterSpec, m *etcdutil.Member, service v1.Service) (string, error) {
+	DNSTimeout := defaultDNSTimeout
+	if cs.Pod != nil {
+		DNSTimeout = cs.Pod.DNSTimeoutInSecond
+	}
+
+	host := ""
+	if cs.ClusteringMode == "discovery" {
+		serviceUrl, err := getServiceHostname(service)
+		if err != nil {
+			return "", err
+		}
+		host = serviceUrl
+	} else {
+		host = m.Addr()
+	}
+	return fmt.Sprintf(`
+		TIMEOUT_READY=%d
+		while ( ! nslookup %s )
+		do
+			# If TIMEOUT_READY is 0 we should never time out and exit
+			TIMEOUT_READY=$(( TIMEOUT_READY-1 ))
+			if [ $TIMEOUT_READY -eq 0 ];
+			then
+				echo "Timed out waiting for DNS entry"
+				exit 1
+			fi
+			sleep 1
+		done`, DNSTimeout, host), nil
+}
+
+func getServiceHostname(service v1.Service) (string, error) {
+	fmt.Printf("Services url list: %v", service.Status.LoadBalancer.Ingress)
+	svcUrl := service.Status.LoadBalancer.Ingress[0].Hostname
+	if svcUrl == "" {
+		return "", fmt.Errorf("failed to get service url: %v", service)
+	} else {
+		return svcUrl, nil
+	}
+}
 func setupEtcdCommand(dataDir string, m *etcdutil.Member, initialCluster string, clusterState string, clusterToken string, clusteringMode string, service v1.Service) (string, error) {
 	if clusteringMode == "discovery" {
-		fmt.Printf("Services url list: %v", service.Status.LoadBalancer.Ingress)
-		serviceUrl := service.Status.LoadBalancer.Ingress[0].Hostname
-		if serviceUrl == "" {
-			return "", fmt.Errorf("failed to get service url: %v", service)
+		serviceUrl, err := getServiceHostname(service)
+		if err != nil {
+			return "", err
 		}
 		command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+
 			"--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+
@@ -543,9 +582,9 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M
 		}})
 	}
 
-	DNSTimeout := defaultDNSTimeout
-	if cs.Pod != nil {
-		DNSTimeout = cs.Pod.DNSTimeoutInSecond
+	initContainerCommand, err := setupInitContainerCommand(cs, m, service)
+	if err != nil {
+		return nil, err
 	}
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
@@ -563,19 +602,7 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M
 				// In etcd 3.2, TLS listener will do a reverse-DNS lookup for pod IP -> hostname.
 				// If DNS entry is not warmed up, it will return empty result and peer connection will be rejected.
 				// In some cases the DNS is not created correctly so we need to time out after a given period.
-				Command: []string{"/bin/sh", "-c", fmt.Sprintf(`
-					TIMEOUT_READY=%d
-					while ( ! nslookup %s )
-					do
-						# If TIMEOUT_READY is 0 we should never time out and exit
-						TIMEOUT_READY=$(( TIMEOUT_READY-1 ))
-                        if [ $TIMEOUT_READY -eq 0 ];
-				        then
-				            echo "Timed out waiting for DNS entry"
-				            exit 1
-				        fi
-						sleep 1
-					done`, DNSTimeout, m.Addr())},
+				Command: []string{"/bin/sh", "-c", initContainerCommand},
 			}},
 			Containers:    []v1.Container{container},
 			RestartPolicy: v1.RestartPolicyNever,

pkg/util/k8sutil/k8sutils_test.go

@@ -82,6 +82,89 @@ func TestEtcdCommandNewLocalCluster(t *testing.T) {
 	}
 }
 
+func TestInitContainerLocalCluster(t *testing.T) {
+	etcdMember := &etcdutil.Member{
+		Name:          "etcd-cluster-test",
+		Namespace:     "etcd",
+		SecurePeer:    false,
+		SecureClient:  false,
+		ClusterDomain: ".local",
+	}
+	clusterSpec := api.ClusterSpec{
+		Size:           1,
+		ClusteringMode: "local",
+		ClusterToken:   "testtoken",
+	}
+	service := v1.Service{}
+
+	initContainerCommand, _ := setupInitContainerCommand(clusterSpec, etcdMember, service)
+
+	expectedCommand := `
+		TIMEOUT_READY=0
+		while ( ! nslookup etcd-cluster-test.etcd-cluster.etcd.svc.local )
+		do
+			# If TIMEOUT_READY is 0 we should never time out and exit
+			TIMEOUT_READY=$(( TIMEOUT_READY-1 ))
+			if [ $TIMEOUT_READY -eq 0 ];
+			then
+				echo "Timed out waiting for DNS entry"
+				exit 1
+			fi
+			sleep 1
+		done`
+
+	if initContainerCommand != expectedCommand {
+		t.Errorf("expected command=%s, got=%s", expectedCommand, initContainerCommand)
+	}
+}
+
+func TestInitContainerDiscoveryCluster(t *testing.T) {
+	etcdMember := &etcdutil.Member{
+		Name:          "etcd-cluster-test",
+		Namespace:     "etcd",
+		SecurePeer:    false,
+		SecureClient:  false,
+		ClusterDomain: ".local",
+	}
+	clusterSpec := api.ClusterSpec{
+		Size:           1,
+		ClusteringMode: "discovery",
+		ClusterToken:   "testtoken",
+	}
+	hostname := v1.LoadBalancerIngress{
+		Hostname: "etcd-peer",
+	}
+	service := v1.Service{
+		Status: v1.ServiceStatus{
+			LoadBalancer: v1.LoadBalancerStatus{
+				Ingress: []v1.LoadBalancerIngress{hostname},
+			},
+		},
+	}
+
+	initContainerCommand, _ := setupInitContainerCommand(clusterSpec, etcdMember, service)
+
+	expectedCommand := `
+		TIMEOUT_READY=0
+		while ( ! nslookup etcd-peer )
+		do
+			# If TIMEOUT_READY is 0 we should never time out and exit
+			TIMEOUT_READY=$(( TIMEOUT_READY-1 ))
+			if [ $TIMEOUT_READY -eq 0 ];
+			then
+				echo "Timed out waiting for DNS entry"
+				exit 1
+			fi
+			sleep 1
+		done`
+
+	if initContainerCommand != expectedCommand {
+		t.Errorf("expected command=%s, got=%s", expectedCommand, initContainerCommand)
+	}
+}
+
+//ToDo: possible failure scenarios for init container command setup
+
 //TODO
 func TestEtcdCommandExistingLocalCluster(t *testing.T) {
 	dataDir := "/var/etcd/data"

pkg/util/k8sutil/pod_util.go

@@ -70,18 +70,22 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con
 
 func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe {
 	// etcd pod is healthy only if it can participate in consensus
-	cmd := "ETCDCTL_API=3 etcdctl endpoint status"
+	var cmd []string
+	cmd = append(cmd, "etcdctl")
 	if isSecure {
-		tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile)
+		cmd = append(cmd, fmt.Sprintf("--endpoints=https://localhost:%d", EtcdClientPort))
 		if isTLSSecret {
-			tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt")
+			cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, "tls.crt"), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, "tls.key"), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, "ca.crt"))
+		} else {
+			cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCertFile), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, etcdutil.CliKeyFile), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCAFile))
 		}
-		cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags)
+
 	}
+	cmd = append(cmd, "endpoint", "status")
 	return &v1.Probe{
 		Handler: v1.Handler{
 			Exec: &v1.ExecAction{
-				Command: []string{"/bin/sh", "-ec", cmd},
+				Command: cmd,
 			},
 		},
 		InitialDelaySeconds: 10,

pkg/util/k8sutil/pod_utils_test.go

@@ -0,0 +1,56 @@
+package k8sutil
+
+import (
+	"testing"
+)
+
+func TestNewEtcdProbe(t *testing.T) {
+	isSecure := false
+	isTLSSecret := false
+	expectedCommand := []string{"etcdctl", "endpoint", "status"}
+
+	probe := newEtcdProbe(isSecure, isTLSSecret)
+
+	if len(probe.Handler.Exec.Command) != len(expectedCommand){
+		t.Errorf("expected command=%s, got=%s", expectedCommand, probe.Handler.Exec.Command)
+	}
+	for i, v := range expectedCommand {
+		if v != probe.Handler.Exec.Command[i] {
+			t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i])
+		}
+	}
+}
+
+func TestNewEtcdProbeWithTLS(t *testing.T) {
+	isSecure := true
+	isTLSSecret := false
+	expectedCommand := []string{"etcdctl", "--endpoints=https://localhost:2379", "--cert=/etc/etcdtls/operator/etcd-tls/etcd-client.crt", "--key=/etc/etcdtls/operator/etcd-tls/etcd-client.key", "--cacert=/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt", "endpoint", "status"}
+
+	probe := newEtcdProbe(isSecure, isTLSSecret)
+
+	if len(probe.Handler.Exec.Command) != len(expectedCommand){
+		t.Errorf("expected command=%v, got=%v", len(expectedCommand), len(probe.Handler.Exec.Command))
+	}
+	for i, v := range expectedCommand {
+		if v != probe.Handler.Exec.Command[i] {
+			t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i])
+		}
+	}
+}
+
+func TestNewEtcdProbeWithTLSWithTLSSecret(t *testing.T) {
+	isSecure := true
+	isTLSSecret := true
+	expectedCommand := []string{"etcdctl", "--endpoints=https://localhost:2379", "--cert=/etc/etcdtls/operator/etcd-tls/tls.crt", "--key=/etc/etcdtls/operator/etcd-tls/tls.key", "--cacert=/etc/etcdtls/operator/etcd-tls/ca.crt", "endpoint", "status"}
+
+	probe := newEtcdProbe(isSecure, isTLSSecret)
+
+	if len(probe.Handler.Exec.Command) != len(expectedCommand){
+		t.Errorf("expected command=%v, got=%v", len(expectedCommand), len(probe.Handler.Exec.Command))
+	}
+	for i, v := range expectedCommand {
+		if v != probe.Handler.Exec.Command[i] {
+			t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i])
+		}
+	}
+}