From 80db5046c0dc3c481a6947a9499e8be77eea81c1 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Wed, 3 May 2023 18:05:21 -0300 Subject: [PATCH 01/18] fix: probes command --- pkg/util/k8sutil/pod_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 22fcce037..2f56af4c9 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -81,7 +81,7 @@ func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-ec", cmd}, + Command: []string{cmd}, }, }, InitialDelaySeconds: 10, From 87e82dbd48be082f58f12d4ffcbb4d3bb1f4e25b Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Thu, 4 May 2023 16:56:55 -0300 Subject: [PATCH 02/18] chore: set minimal version to v3.4.19 --- README.md | 4 ++-- example/example-etcd-cluster.yaml | 2 +- pkg/apis/etcd/v1beta2/cluster.go | 6 +++--- test/e2e/basic_test.go | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 380a46cd7..264b6960e 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ See the [Resources and Labels](./doc/user/resource_labels.md) doc for an overvie ## Requirements - Kubernetes 1.8+ -- etcd 3.2.13+ +- etcd 3.3+ ## Demo @@ -126,7 +126,7 @@ metadata: name: "example-etcd-cluster" spec: size: 3 - version: "v3.2.13" + version: "v3.4.19" ``` ``` $ kubectl apply -f example/example-etcd-cluster.yaml diff --git a/example/example-etcd-cluster.yaml b/example/example-etcd-cluster.yaml index 1c2169751..7d9e29c7a 100644 --- a/example/example-etcd-cluster.yaml +++ b/example/example-etcd-cluster.yaml @@ -8,4 +8,4 @@ metadata: # etcd.database.coreos.com/scope: clusterwide spec: size: 3 - version: "v3.2.13" + version: "v3.4.19" diff --git a/pkg/apis/etcd/v1beta2/cluster.go b/pkg/apis/etcd/v1beta2/cluster.go index 24be6cd1a..ada9b90a3 100644 --- a/pkg/apis/etcd/v1beta2/cluster.go +++ b/pkg/apis/etcd/v1beta2/cluster.go @@ -24,7 +24,7 @@ import ( const ( defaultRepository = "quay.io/coreos/etcd" - DefaultEtcdVersion = "v3.2.13" + DefaultEtcdVersion = "v3.4.19" ) var ( @@ -83,10 +83,10 @@ type ClusterSpec struct { // The etcd-operator will eventually make the etcd cluster version // equal to the expected version. // - // The version must follow the [semver]( http://semver.org) format, for example "3.2.13". + // The version must follow the [semver]( http://semver.org) format, for example "v3.4.19". // Only etcd released versions are supported: https://github.com/coreos/etcd/releases // - // If version is not set, default is "v3.2.13". + // If version is not set, default is "v3.4.19". Version string `json:"version,omitempty"` // Paused is to pause the control of the operator for the etcd cluster. diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 1158fe505..4773c6463 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -108,7 +108,7 @@ func TestEtcdUpgrade(t *testing.T) { } f := framework.Global origEtcd := e2eutil.NewCluster("test-etcd-", 3) - origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.1.10") + origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.4.19") origEtcd.Spec.Repository = "quay.io/coreos/etcd" testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, origEtcd) if err != nil { @@ -121,12 +121,12 @@ func TestEtcdUpgrade(t *testing.T) { } }() - err = e2eutil.WaitSizeAndVersionReached(t, context.Background(), f.KubeClient, "v3.1.10", 3, f.RetryAttempts, testEtcd) + err = e2eutil.WaitSizeAndVersionReached(t, context.Background(), f.KubeClient, "v3.4.19", 3, f.RetryAttempts, testEtcd) if err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } - targetVersion := "v3.2.13" + targetVersion := "v3.4.20" updateFunc := func(cl *api.EtcdCluster) { cl = e2eutil.ClusterWithVersion(cl, targetVersion) } From 9e72a62e34fd3228daaf980367cdbceda9b419b3 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Thu, 4 May 2023 17:38:09 -0300 Subject: [PATCH 03/18] test removing etcd version var --- pkg/util/k8sutil/pod_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 2f56af4c9..63595da25 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -70,13 +70,13 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus - cmd := "ETCDCTL_API=3 etcdctl endpoint status" + cmd := "etcdctl endpoint status" if isSecure { tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile) if isTLSSecret { tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt") } - cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) + cmd = fmt.Sprintf("etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) } return &v1.Probe{ Handler: v1.Handler{ From 0897bdcf76a2b2364cd5bc040134fb05f3ff883a Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Thu, 4 May 2023 18:09:05 -0300 Subject: [PATCH 04/18] test rollback with newer etcd version --- pkg/util/k8sutil/pod_util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 63595da25..22fcce037 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -70,18 +70,18 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus - cmd := "etcdctl endpoint status" + cmd := "ETCDCTL_API=3 etcdctl endpoint status" if isSecure { tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile) if isTLSSecret { tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt") } - cmd = fmt.Sprintf("etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) + cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) } return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{cmd}, + Command: []string{"/bin/sh", "-ec", cmd}, }, }, InitialDelaySeconds: 10, From 534436bef9951f25e171ab57fc5e05a09cf5dc70 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 5 May 2023 11:45:38 -0300 Subject: [PATCH 05/18] re-apply probe change --- pkg/util/k8sutil/pod_util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 22fcce037..63595da25 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -70,18 +70,18 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus - cmd := "ETCDCTL_API=3 etcdctl endpoint status" + cmd := "etcdctl endpoint status" if isSecure { tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile) if isTLSSecret { tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt") } - cmd = fmt.Sprintf("ETCDCTL_API=3 etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) + cmd = fmt.Sprintf("etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) } return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{"/bin/sh", "-ec", cmd}, + Command: []string{cmd}, }, }, InitialDelaySeconds: 10, From 73bcbe3f7831f072bd40e48241b6979ef8f6a4f9 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 5 May 2023 13:30:46 -0300 Subject: [PATCH 06/18] using etcdctl from usr folder --- pkg/util/k8sutil/pod_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 63595da25..53ae3efc0 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -81,7 +81,7 @@ func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{cmd}, + Command: []string{"/usr/local/bin/", cmd}, }, }, InitialDelaySeconds: 10, From d0ea7f3422203a3c9d55716794ca9147e2f5b82f Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 5 May 2023 13:46:23 -0300 Subject: [PATCH 07/18] fix command --- pkg/util/k8sutil/pod_util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 53ae3efc0..c77baae76 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -70,18 +70,18 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus - cmd := "etcdctl endpoint status" + cmd := "endpoint status" if isSecure { tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile) if isTLSSecret { tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt") } - cmd = fmt.Sprintf("etcdctl --endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) + cmd = fmt.Sprintf("--endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) } return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{"/usr/local/bin/", cmd}, + Command: []string{"/usr/local/bin/etcdctl", cmd}, }, }, InitialDelaySeconds: 10, From 88322808ebfadb38995f54a541d669847f86ae20 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 5 May 2023 14:48:32 -0300 Subject: [PATCH 08/18] format command correctly --- pkg/util/k8sutil/pod_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index c77baae76..4c7cbecfb 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -81,7 +81,7 @@ func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{"/usr/local/bin/etcdctl", cmd}, + Command: []string{fmt.Sprintf("/usr/local/bin/etcdctl %s", cmd)}, }, }, InitialDelaySeconds: 10, From b492ed08bd6831f2d83aeb2e04a13ed479223499 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 12 May 2023 14:39:19 -0300 Subject: [PATCH 09/18] fix: init container checking service dns before startup --- pkg/util/k8sutil/k8sutil.go | 77 ++++++++++++++++++++-------- pkg/util/k8sutil/k8sutils_test.go | 83 +++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 20 deletions(-) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 9eac9d7f2..a572f4e63 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -423,12 +423,61 @@ func setupClientServiceURL(endpoint string) string { return fmt.Sprintf("http://%s:2379", endpoint) } +func setupInitContainerCommand(cs api.ClusterSpec, m *etcdutil.Member, service v1.Service) (string, error) { + DNSTimeout := defaultDNSTimeout + if cs.Pod != nil { + DNSTimeout = cs.Pod.DNSTimeoutInSecond + } + + if cs.ClusteringMode == "discovery" { + serviceUrl, err := getServiceHostname(service) + if err != nil { + return "", err + } + return fmt.Sprintf(` + TIMEOUT_READY=%d + while ( ! nslookup %s ) + do + # If TIMEOUT_READY is 0 we should never time out and exit + TIMEOUT_READY=$(( TIMEOUT_READY-1 )) + if [ $TIMEOUT_READY -eq 0 ]; + then + echo "Timed out waiting for DNS entry" + exit 1 + fi + sleep 1 + done`, DNSTimeout, serviceUrl), nil + } else { + return fmt.Sprintf(` + TIMEOUT_READY=%d + while ( ! nslookup %s ) + do + # If TIMEOUT_READY is 0 we should never time out and exit + TIMEOUT_READY=$(( TIMEOUT_READY-1 )) + if [ $TIMEOUT_READY -eq 0 ]; + then + echo "Timed out waiting for DNS entry" + exit 1 + fi + sleep 1 + done`, DNSTimeout, m.Addr()), nil + } +} + +func getServiceHostname(service v1.Service) (string, error) { + fmt.Printf("Services url list: %v", service.Status.LoadBalancer.Ingress) + svcUrl := service.Status.LoadBalancer.Ingress[0].Hostname + if svcUrl == "" { + return "", fmt.Errorf("failed to get service url: %v", service) + } else { + return svcUrl, nil + } +} func setupEtcdCommand(dataDir string, m *etcdutil.Member, initialCluster string, clusterState string, clusterToken string, clusteringMode string, service v1.Service) (string, error) { if clusteringMode == "discovery" { - fmt.Printf("Services url list: %v", service.Status.LoadBalancer.Ingress) - serviceUrl := service.Status.LoadBalancer.Ingress[0].Hostname - if serviceUrl == "" { - return "", fmt.Errorf("failed to get service url: %v", service) + serviceUrl, err := getServiceHostname(service) + if err != nil { + return "", err } command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+ "--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+ @@ -543,9 +592,9 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M }}) } - DNSTimeout := defaultDNSTimeout - if cs.Pod != nil { - DNSTimeout = cs.Pod.DNSTimeoutInSecond + initContainerCommand, err := setupInitContainerCommand(cs, m, service) + if err != nil { + return nil, err } pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -563,19 +612,7 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M // In etcd 3.2, TLS listener will do a reverse-DNS lookup for pod IP -> hostname. // If DNS entry is not warmed up, it will return empty result and peer connection will be rejected. // In some cases the DNS is not created correctly so we need to time out after a given period. - Command: []string{"/bin/sh", "-c", fmt.Sprintf(` - TIMEOUT_READY=%d - while ( ! nslookup %s ) - do - # If TIMEOUT_READY is 0 we should never time out and exit - TIMEOUT_READY=$(( TIMEOUT_READY-1 )) - if [ $TIMEOUT_READY -eq 0 ]; - then - echo "Timed out waiting for DNS entry" - exit 1 - fi - sleep 1 - done`, DNSTimeout, m.Addr())}, + Command: []string{"/bin/sh", "-c", initContainerCommand}, }}, Containers: []v1.Container{container}, RestartPolicy: v1.RestartPolicyNever, diff --git a/pkg/util/k8sutil/k8sutils_test.go b/pkg/util/k8sutil/k8sutils_test.go index 5c92bf43c..0864b47d0 100644 --- a/pkg/util/k8sutil/k8sutils_test.go +++ b/pkg/util/k8sutil/k8sutils_test.go @@ -82,6 +82,89 @@ func TestEtcdCommandNewLocalCluster(t *testing.T) { } } +func TestInitContainerLocalCluster(t *testing.T) { + etcdMember := &etcdutil.Member{ + Name: "etcd-cluster-test", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + clusterSpec := api.ClusterSpec{ + Size: 1, + ClusteringMode: "local", + ClusterToken: "testtoken", + } + service := v1.Service{} + + initContainerCommand, _ := setupInitContainerCommand(clusterSpec, etcdMember, service) + + expectedCommand := ` + TIMEOUT_READY=0 + while ( ! nslookup etcd-cluster-test.etcd-cluster.etcd.svc.local ) + do + # If TIMEOUT_READY is 0 we should never time out and exit + TIMEOUT_READY=$(( TIMEOUT_READY-1 )) + if [ $TIMEOUT_READY -eq 0 ]; + then + echo "Timed out waiting for DNS entry" + exit 1 + fi + sleep 1 + done` + + if initContainerCommand != expectedCommand { + t.Errorf("expected command=%s, got=%s", expectedCommand, initContainerCommand) + } +} + +func TestInitContainerDiscoveryCluster(t *testing.T) { + etcdMember := &etcdutil.Member{ + Name: "etcd-cluster-test", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + clusterSpec := api.ClusterSpec{ + Size: 1, + ClusteringMode: "discovery", + ClusterToken: "testtoken", + } + hostname := v1.LoadBalancerIngress{ + Hostname: "etcd-peer", + } + service := v1.Service{ + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{hostname}, + }, + }, + } + + initContainerCommand, _ := setupInitContainerCommand(clusterSpec, etcdMember, service) + + expectedCommand := ` + TIMEOUT_READY=0 + while ( ! nslookup etcd-peer ) + do + # If TIMEOUT_READY is 0 we should never time out and exit + TIMEOUT_READY=$(( TIMEOUT_READY-1 )) + if [ $TIMEOUT_READY -eq 0 ]; + then + echo "Timed out waiting for DNS entry" + exit 1 + fi + sleep 1 + done` + + if initContainerCommand != expectedCommand { + t.Errorf("expected command=%s, got=%s", expectedCommand, initContainerCommand) + } +} + +//ToDo: possible failure scenarios for init container command setup + //TODO func TestEtcdCommandExistingLocalCluster(t *testing.T) { dataDir := "/var/etcd/data" From 87c7b79f1e9d26158c87eaa1c2939e321db91457 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 12 May 2023 15:33:59 -0300 Subject: [PATCH 10/18] chore: remove repeated code --- pkg/util/k8sutil/k8sutil.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index a572f4e63..1b65ce8a3 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -429,26 +429,17 @@ func setupInitContainerCommand(cs api.ClusterSpec, m *etcdutil.Member, service v DNSTimeout = cs.Pod.DNSTimeoutInSecond } + host := "" if cs.ClusteringMode == "discovery" { serviceUrl, err := getServiceHostname(service) if err != nil { return "", err } - return fmt.Sprintf(` - TIMEOUT_READY=%d - while ( ! nslookup %s ) - do - # If TIMEOUT_READY is 0 we should never time out and exit - TIMEOUT_READY=$(( TIMEOUT_READY-1 )) - if [ $TIMEOUT_READY -eq 0 ]; - then - echo "Timed out waiting for DNS entry" - exit 1 - fi - sleep 1 - done`, DNSTimeout, serviceUrl), nil + host = serviceUrl } else { - return fmt.Sprintf(` + host = m.Addr() + } + return fmt.Sprintf(` TIMEOUT_READY=%d while ( ! nslookup %s ) do @@ -460,8 +451,7 @@ func setupInitContainerCommand(cs api.ClusterSpec, m *etcdutil.Member, service v exit 1 fi sleep 1 - done`, DNSTimeout, m.Addr()), nil - } + done`, DNSTimeout, host), nil } func getServiceHostname(service v1.Service) (string, error) { From 1e90d07bc5add1b8987c06c13c33bddee4c63a27 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Thu, 18 May 2023 18:44:01 -0300 Subject: [PATCH 11/18] chore: probe commands must be a list of strings --- pkg/util/k8sutil/pod_util.go | 14 +++++--- pkg/util/k8sutil/pod_utils_test.go | 56 ++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 pkg/util/k8sutil/pod_utils_test.go diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 4c7cbecfb..13c54b8da 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -70,18 +70,22 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus - cmd := "endpoint status" + var cmd []string + cmd = append(cmd, "etcdctl") if isSecure { - tlsFlags := fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, etcdutil.CliCertFile, etcdutil.CliKeyFile, etcdutil.CliCAFile) + cmd = append(cmd, fmt.Sprintf("--endpoints=https://localhost:%d", EtcdClientPort)) if isTLSSecret { - tlsFlags = fmt.Sprintf("--cert=%[1]s/%[2]s --key=%[1]s/%[3]s --cacert=%[1]s/%[4]s", operatorEtcdTLSDir, "tls.crt", "tls.key", "ca.crt") + cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, "tls.crt"), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, "tls.key"), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, "ca.crt")) + } else { + cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCertFile), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, etcdutil.CliKeyFile), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCAFile)) } - cmd = fmt.Sprintf("--endpoints=https://localhost:%d %s endpoint status", EtcdClientPort, tlsFlags) + } + cmd = append(cmd, "endpoint", "status") return &v1.Probe{ Handler: v1.Handler{ Exec: &v1.ExecAction{ - Command: []string{fmt.Sprintf("/usr/local/bin/etcdctl %s", cmd)}, + Command: cmd, }, }, InitialDelaySeconds: 10, diff --git a/pkg/util/k8sutil/pod_utils_test.go b/pkg/util/k8sutil/pod_utils_test.go new file mode 100644 index 000000000..62c2bb9f2 --- /dev/null +++ b/pkg/util/k8sutil/pod_utils_test.go @@ -0,0 +1,56 @@ +package k8sutil + +import ( + "testing" +) + +func TestNewEtcdProbe(t *testing.T) { + isSecure := false + isTLSSecret := false + expectedCommand := []string{"etcdctl", "endpoint", "status"} + + probe := newEtcdProbe(isSecure, isTLSSecret) + + if len(probe.Handler.Exec.Command) != len(expectedCommand){ + t.Errorf("expected command=%s, got=%s", expectedCommand, probe.Handler.Exec.Command) + } + for i, v := range expectedCommand { + if v != probe.Handler.Exec.Command[i] { + t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i]) + } + } +} + +func TestNewEtcdProbeWithTLS(t *testing.T) { + isSecure := true + isTLSSecret := false + expectedCommand := []string{"etcdctl", "--endpoints=https://localhost:2379", "--cert=/etc/etcdtls/operator/etcd-tls/etcd-client.crt", "--key=/etc/etcdtls/operator/etcd-tls/etcd-client.key", "--cacert=/etc/etcdtls/operator/etcd-tls/etcd-client-ca.crt", "endpoint", "status"} + + probe := newEtcdProbe(isSecure, isTLSSecret) + + if len(probe.Handler.Exec.Command) != len(expectedCommand){ + t.Errorf("expected command=%v, got=%v", len(expectedCommand), len(probe.Handler.Exec.Command)) + } + for i, v := range expectedCommand { + if v != probe.Handler.Exec.Command[i] { + t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i]) + } + } +} + +func TestNewEtcdProbeWithTLSWithTLSSecret(t *testing.T) { + isSecure := true + isTLSSecret := true + expectedCommand := []string{"etcdctl", "--endpoints=https://localhost:2379", "--cert=/etc/etcdtls/operator/etcd-tls/tls.crt", "--key=/etc/etcdtls/operator/etcd-tls/tls.key", "--cacert=/etc/etcdtls/operator/etcd-tls/ca.crt", "endpoint", "status"} + + probe := newEtcdProbe(isSecure, isTLSSecret) + + if len(probe.Handler.Exec.Command) != len(expectedCommand){ + t.Errorf("expected command=%v, got=%v", len(expectedCommand), len(probe.Handler.Exec.Command)) + } + for i, v := range expectedCommand { + if v != probe.Handler.Exec.Command[i] { + t.Errorf("expected piece=%s, got=%s", v, probe.Handler.Exec.Command[i]) + } + } +} \ No newline at end of file From 7624fc80afbd2cccbd058f9424d516794b94a6e8 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Thu, 18 May 2023 18:48:30 -0300 Subject: [PATCH 12/18] e2e test for discovery cluster --- test/e2e/basic_test.go | 44 ++++++++++++++++++++++++++++++++++- test/e2e/e2eutil/spec_util.go | 5 ++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 4773c6463..21e15ba09 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -19,6 +19,8 @@ import ( "os" "testing" "time" + "net/http" + "io/ioutil" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/test/e2e/e2eutil" @@ -46,6 +48,46 @@ func TestCreateCluster(t *testing.T) { } } +func getDiscoveryToken(t *testing.T) string { + resp, err := http.Get("https://discovery.etcd.io/new?size=1") + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func TestCreateClusterDiscovery(t *testing.T) { + if os.Getenv(envParallelTest) == envParallelTestTrue { + t.Parallel() + } + f := framework.Global + cluster := e2eutil.NewCluster("test-etcd-", 3) + cluster = e2eutil.ClusterWithVersion(cluster, "v3.5.7") + cluster = e2eutil.ClusterWithRepo(cluster, "quay.io/coreos/etcd") + token := getDiscoveryToken(t) + cluster.Spec.ClusteringMode = "discovery" + cluster.Spec.ClusterToken = token + + testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, cluster) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := e2eutil.DeleteCluster(t, context.Background(), f.CRClient, f.KubeClient, testEtcd); err != nil { + t.Fatal(err) + } + }() + + if _, err := e2eutil.WaitUntilSizeReached(t, context.Background(), f.CRClient, 1, f.RetryAttempts, testEtcd); err != nil { + t.Fatalf("failed to create 1 member etcd cluster: %v", err) + } +} + // TestPauseControl tests the user can pause the operator from controlling // an etcd cluster. func TestPauseControl(t *testing.T) { @@ -109,7 +151,7 @@ func TestEtcdUpgrade(t *testing.T) { f := framework.Global origEtcd := e2eutil.NewCluster("test-etcd-", 3) origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.4.19") - origEtcd.Spec.Repository = "quay.io/coreos/etcd" + origEtcd = e2eutil.ClusterWithRepo(origEtcd, "quay.io/coreos/etcd") testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, origEtcd) if err != nil { t.Fatal(err) diff --git a/test/e2e/e2eutil/spec_util.go b/test/e2e/e2eutil/spec_util.go index b65f535ad..73e4290df 100644 --- a/test/e2e/e2eutil/spec_util.go +++ b/test/e2e/e2eutil/spec_util.go @@ -111,3 +111,8 @@ func ClusterWithVersion(cl *api.EtcdCluster, version string) *api.EtcdCluster { func NameLabelSelector(name string) map[string]string { return map[string]string{"name": name} } + +func ClusterWithRepo(cl *api.EtcdCluster, repo string) *api.EtcdCluster { + cl.Spec.Repository = repo + return cl +} \ No newline at end of file From 2d2eb56e950be3ca024419cb5df76be9501fe9c2 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 19 May 2023 09:28:45 -0300 Subject: [PATCH 13/18] fix: discovery test --- test/e2e/basic_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 21e15ba09..f9e0f7392 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -65,7 +65,7 @@ func TestCreateClusterDiscovery(t *testing.T) { t.Parallel() } f := framework.Global - cluster := e2eutil.NewCluster("test-etcd-", 3) + cluster := e2eutil.NewCluster("test-etcd-", 1) cluster = e2eutil.ClusterWithVersion(cluster, "v3.5.7") cluster = e2eutil.ClusterWithRepo(cluster, "quay.io/coreos/etcd") token := getDiscoveryToken(t) From 238e458cc1eec04fbfc902d2297c412f1907d4bc Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 19 May 2023 09:58:30 -0300 Subject: [PATCH 14/18] remove discovery test :( --- test/e2e/basic_test.go | 79 +++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index f9e0f7392..e163aeb20 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -48,45 +48,46 @@ func TestCreateCluster(t *testing.T) { } } -func getDiscoveryToken(t *testing.T) string { - resp, err := http.Get("https://discovery.etcd.io/new?size=1") - if err != nil { - t.Fatal(err) - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - return string(body) -} - -func TestCreateClusterDiscovery(t *testing.T) { - if os.Getenv(envParallelTest) == envParallelTestTrue { - t.Parallel() - } - f := framework.Global - cluster := e2eutil.NewCluster("test-etcd-", 1) - cluster = e2eutil.ClusterWithVersion(cluster, "v3.5.7") - cluster = e2eutil.ClusterWithRepo(cluster, "quay.io/coreos/etcd") - token := getDiscoveryToken(t) - cluster.Spec.ClusteringMode = "discovery" - cluster.Spec.ClusterToken = token - - testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, cluster) - if err != nil { - t.Fatal(err) - } - - defer func() { - if err := e2eutil.DeleteCluster(t, context.Background(), f.CRClient, f.KubeClient, testEtcd); err != nil { - t.Fatal(err) - } - }() - - if _, err := e2eutil.WaitUntilSizeReached(t, context.Background(), f.CRClient, 1, f.RetryAttempts, testEtcd); err != nil { - t.Fatalf("failed to create 1 member etcd cluster: %v", err) - } -} +// Test not used cause the testing cluster doesn't have aws lb controller, which this version uses +// func getDiscoveryToken(t *testing.T) string { +// resp, err := http.Get("https://discovery.etcd.io/new?size=1") +// if err != nil { +// t.Fatal(err) +// } +// body, err := ioutil.ReadAll(resp.Body) +// if err != nil { +// t.Fatal(err) +// } +// return string(body) +// } + +// func TestCreateClusterDiscovery(t *testing.T) { +// if os.Getenv(envParallelTest) == envParallelTestTrue { +// t.Parallel() +// } +// f := framework.Global +// cluster := e2eutil.NewCluster("test-etcd-", 1) +// cluster = e2eutil.ClusterWithVersion(cluster, "v3.5.7") +// cluster = e2eutil.ClusterWithRepo(cluster, "quay.io/coreos/etcd") +// token := getDiscoveryToken(t) +// cluster.Spec.ClusteringMode = "discovery" +// cluster.Spec.ClusterToken = token + +// testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, cluster) +// if err != nil { +// t.Fatal(err) +// } + +// defer func() { +// if err := e2eutil.DeleteCluster(t, context.Background(), f.CRClient, f.KubeClient, testEtcd); err != nil { +// t.Fatal(err) +// } +// }() + +// if _, err := e2eutil.WaitUntilSizeReached(t, context.Background(), f.CRClient, 1, f.RetryAttempts, testEtcd); err != nil { +// t.Fatalf("failed to create 1 member etcd cluster: %v", err) +// } +// } // TestPauseControl tests the user can pause the operator from controlling // an etcd cluster. From 199a255dc25a96fd2496c2f608e995778e5372ac Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Fri, 19 May 2023 10:15:13 -0300 Subject: [PATCH 15/18] fix imports --- test/e2e/basic_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index e163aeb20..08671f729 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -19,8 +19,8 @@ import ( "os" "testing" "time" - "net/http" - "io/ioutil" + // "net/http" + // "io/ioutil" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/test/e2e/e2eutil" From 090d1b4198fa9b1cf9f21374a2cd912cc94e45cb Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Mon, 22 May 2023 14:28:02 -0300 Subject: [PATCH 16/18] Test for distroless versions --- test/e2e/basic_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 08671f729..40a883e14 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -48,6 +48,30 @@ func TestCreateCluster(t *testing.T) { } } +func TestCreateClusterDistrolessVersion(t *testing.T) { + if os.Getenv(envParallelTest) == envParallelTestTrue { + t.Parallel() + } + f := framework.Global + origEtcd := e2eutil.NewCluster("test-etcd-", 3) + origEtcd = e2eutil.ClusterWithVersion(origEtcd, "v3.5.7") + origEtcd = e2eutil.ClusterWithRepo(origEtcd, "quay.io/coreos/etcd") + testEtcd, err := e2eutil.CreateCluster(t, context.Background(), f.CRClient, f.Namespace, origEtcd) + if err != nil { + t.Fatal(err) + } + + defer func() { + if err := e2eutil.DeleteCluster(t, context.Background(), f.CRClient, f.KubeClient, testEtcd); err != nil { + t.Fatal(err) + } + }() + + if _, err := e2eutil.WaitUntilSizeReached(t, context.Background(), f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil { + t.Fatalf("failed to create 3 members etcd cluster: %v", err) + } +} + // Test not used cause the testing cluster doesn't have aws lb controller, which this version uses // func getDiscoveryToken(t *testing.T) string { // resp, err := http.Get("https://discovery.etcd.io/new?size=1") From bf6b37f1cb6eb72ab5f23eaaea0757a5a94a8c9c Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Mon, 22 May 2023 15:13:21 -0300 Subject: [PATCH 17/18] unnecessary comment --- test/e2e/basic_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 40a883e14..c665ee8ea 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -19,8 +19,6 @@ import ( "os" "testing" "time" - // "net/http" - // "io/ioutil" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" "github.com/coreos/etcd-operator/test/e2e/e2eutil" From bf29ceed28f44b3f5924f434bd03dd9ba6f61480 Mon Sep 17 00:00:00 2001 From: Carlos Martins Date: Mon, 22 May 2023 15:43:27 -0300 Subject: [PATCH 18/18] code improvement on cert files --- pkg/util/k8sutil/pod_util.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/util/k8sutil/pod_util.go b/pkg/util/k8sutil/pod_util.go index 13c54b8da..be8da8f9c 100644 --- a/pkg/util/k8sutil/pod_util.go +++ b/pkg/util/k8sutil/pod_util.go @@ -71,15 +71,22 @@ func containerWithRequirements(c v1.Container, r v1.ResourceRequirements) v1.Con func newEtcdProbe(isSecure, isTLSSecret bool) *v1.Probe { // etcd pod is healthy only if it can participate in consensus var cmd []string + var certFile string + var keyFile string + var cacertFile string cmd = append(cmd, "etcdctl") if isSecure { cmd = append(cmd, fmt.Sprintf("--endpoints=https://localhost:%d", EtcdClientPort)) if isTLSSecret { - cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, "tls.crt"), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, "tls.key"), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, "ca.crt")) + certFile = "tls.crt" + keyFile = "tls.key" + cacertFile = "ca.crt" } else { - cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCertFile), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, etcdutil.CliKeyFile), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, etcdutil.CliCAFile)) + certFile = etcdutil.CliCertFile + keyFile = etcdutil.CliKeyFile + cacertFile = etcdutil.CliCAFile } - + cmd = append(cmd, fmt.Sprintf("--cert=%s/%s", operatorEtcdTLSDir, certFile), fmt.Sprintf("--key=%s/%s", operatorEtcdTLSDir, keyFile), fmt.Sprintf("--cacert=%s/%s", operatorEtcdTLSDir, cacertFile)) } cmd = append(cmd, "endpoint", "status") return &v1.Probe{