fix: etcd 3.5.7 compatibility (do not depend on /bin/sh)

Some containers where started with or had liveness checks with /bin/sh.
This fails starting with etcd 3.5.7 as it does not come with any shell.

As a consequence I also had to raise the etcd compatibility to at least
3.3 as this is the first version which has ETCDCTL_API=3 as default.

This may be solved in a different way, but 3.2 should not be used anymore.

README.md

@@ -36,7 +36,7 @@ See the [Resources and Labels](./doc/user/resource_labels.md) doc for an overvie
 ## Requirements
 
 - Kubernetes 1.23+ (may work perfectly fine on older versions just no effort will be made to be compatible)
-- etcd 3.2.13+
+- etcd 3.3.25+
 
 ## Demo
 

doc/user/client_service.md

@@ -15,8 +15,7 @@ The client service is of type `ClusterIP` and accessible only from within the Ku
 For example, access the service from a pod in the cluster:
 
 ```
-$ kubectl run --rm -i --tty fun --image quay.io/coreos/etcd --restart=Never -- /bin/sh
-/ # ETCDCTL_API=3 etcdctl --endpoints http://example-etcd-cluster-client:2379 put foo bar
+$ kubectl run --rm  --tty fun --image quay.io/coreos/etcd --restart=Never -- etcdctl --endpoints http://example-etcd-cluster-client:2379 put foo bar
 OK
 (ctrl-D to exit)
 ```

pkg/util/k8sutil/k8sutil.go

@@ -99,6 +99,13 @@ func PVCNameFromMember(memberName string) string {
 }
 
 func makeRestoreInitContainers(backupURL *url.URL, token, repo, version string, m *etcdutil.Member) []v1.Container {
+	cmd := fmt.Sprintf("etcdctl snapshot restore %[1]s"+
+		" --name %[2]s"+
+		" --initial-cluster %[2]s=%[3]s"+
+		" --initial-cluster-token %[4]s"+
+		" --initial-advertise-peer-urls %[3]s"+
+		" --data-dir %[5]s 2>/dev/termination-log", backupFile, m.Name, m.PeerURL(), token, dataDir)
+
 	return []v1.Container{
 		{
 			Name:  "fetch-backup",
@@ -117,17 +124,9 @@ fi
 			VolumeMounts: etcdVolumeMounts(),
 		},
 		{
-			Name:  "restore-datadir",
-			Image: ImageName(repo, version),
-			Command: []string{
-				"/bin/sh", "-ec",
-				fmt.Sprintf("ETCDCTL_API=3 etcdctl snapshot restore %[1]s"+
-					" --name %[2]s"+
-					" --initial-cluster %[2]s=%[3]s"+
-					" --initial-cluster-token %[4]s"+
-					" --initial-advertise-peer-urls %[3]s"+
-					" --data-dir %[5]s 2>/dev/termination-log", backupFile, m.Name, m.PeerURL(), token, dataDir),
-			},
+			Name:         "restore-datadir",
+			Image:        ImageName(repo, version),
+			Command:      strings.Split(cmd, " "),
 			VolumeMounts: etcdVolumeMounts(),
 		},
 	}

pkg/util/k8sutil/pod_util.go

@@ -17,6 +17,7 @@ package k8sutil
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	api "github.com/on2itsecurity/etcd-operator/pkg/apis/etcd/v1beta2"
 	"github.com/on2itsecurity/etcd-operator/pkg/util/etcdutil"
@@ -70,18 +71,18 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con
 
 func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe {
 	// etcd pod is healthy only if it can participate in consensus
-	cmd := "ETCDCTL_API=3 etcdctl endpoint status"
+	cmd := "etcdctl endpoint status"
 	if isSecure {
 		tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile)
 		if isTLSSecret {
 			tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt")
 		}
-		cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags)
+		cmd = fmt.Sprintf("etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags)
 	}
 	return &v1.Probe{
 		ProbeHandler: v1.ProbeHandler{
 			Exec: &v1.ExecAction{
-				Command: []string{"/bin/sh", "-ec", cmd},
+				Command: strings.Split(cmd, " "),
 			},
 		},
 		InitialDelaySeconds: 10,

test/e2e/basic_test.go

@@ -108,7 +108,7 @@ func TestEtcdUpgrade(t *testing.T) {
 	}
 	f := framework.Global
 	origEtcd := e2eutil.NewCluster("test-etcd-", 3)
-	origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.2.13")
+	origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.3.25")
 	origEtcd.Spec.Repository = "gcr.io/etcd-development/etcd"
 	testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, origEtcd)
 	if err != nil {
@@ -121,7 +121,7 @@ func TestEtcdUpgrade(t *testing.T) {
 		}
 	}()
 
-	err = e2eutil.WaitSizeAndVersionReached(t, context.Background(), f.KubeClient, "v3.2.13", 3, f.RetryAttempts, testEtcd)
+	err = e2eutil.WaitSizeAndVersionReached(t, context.Background(), f.KubeClient, "v3.3.25", 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}