Merge pull request #3 from topfreegames/feat/distributed-cluster

Feat/discovery mode for distributed cluster

.github/workflows/go-build.yml

@@ -1,6 +1,5 @@
 name: Build
-on:
-  push:
+on: [push, pull_request]
 jobs:
   operator:
     name: Operator

.github/workflows/go-test-e2e.yml

@@ -1,6 +1,5 @@
 name: Testing E2E
-on:
-  push:
+on: [push, pull_request]
 jobs:
   prepare:
     name: Prepare
@@ -14,26 +13,41 @@ jobs:
 
       - name: Check out code into the Go module directory
         uses: actions/[email protected]
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
 
-      - name: Docker login
-        run: docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
-        env:
-          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Prepare e2e image
-        run: |
-          hack/build/e2e/docker_push
-        env:
-          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
-
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: .
+          file: test/pod/Dockerfile
+          tags: tfgco/etcd-operator-e2e:${{github.sha}}
+
       - name: Prepare operator image
         run: |
           hack/build/operator/build
           hack/build/backup-operator/build
           hack/build/restore-operator/build
-          IMAGE=${OPERATOR_IMAGE} hack/build/docker_push
-        env:
-          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+      
+      - name: Prepare operator image
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: .
+          file: hack/build/Dockerfile
+          tags: tfgco/etcd-operator:${{github.sha}}
+
   test-e2e:
     name: E2E
     runs-on: ubuntu-latest
@@ -48,12 +62,19 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/[email protected]
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: KinD (Kubernetes in Docker) Action
         uses: engineerd/[email protected]
-
+        with: 
+          version: "v0.17.0"
+
       - name: Test
         run: |
-          docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
           docker pull $TEST_IMAGE
           docker pull $OPERATOR_IMAGE
           export KUBECONFIG="${HOME}/.kube/config"
@@ -62,13 +83,13 @@ jobs:
           hack/ci/run_e2e
         env:
           GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
-          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+          OPERATOR_IMAGE: tfgco/etcd-operator:${{github.sha}}
           TEST_AWS_SECRET: na
           TEST_S3_BUCKET: na
           TEST_NAMESPACE: default
           BUILD_IMAGE: false
           BUILD_E2E: false
-          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+          TEST_IMAGE: tfgco/etcd-operator-e2e:${{github.sha}}
           PASSES: e2e
 
       - name: Show logs

.github/workflows/go-test.yml

@@ -1,6 +1,5 @@
 name: Testing
-on:
-  push:
+on: [push, pull_request]
 jobs:
 
   build:

example/rbac/cluster-role-binding-template.yaml

@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: <ROLE_BINDING_NAME>

example/rbac/cluster-role-template.yaml

@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: <ROLE_NAME>

pkg/apis/etcd/v1beta2/cluster.go

@@ -102,6 +102,9 @@ type ClusterSpec struct {
 
 	// etcd cluster TLS configuration
 	TLS *TLSPolicy `json:"TLS,omitempty"`
+
+	ClusteringMode string `json:"clusteringMode,omitempty"`
+	ClusterToken string `json:"clusterToken,omitempty"`
 }
 
 // PodPolicy defines the policy to create pod for the etcd container.

pkg/controller/restore-operator/sync.go

@@ -225,6 +225,9 @@ func (r *Restore) createSeedMember(ctx context.Context, ec *api.EtcdCluster, svc
 	backupURL := backupapi.BackupURLForRestore("http", svcAddr, clusterName, namespace)
 	ec.SetDefaults()
 	pod, err := k8sutil.NewSeedMemberPod(ctx, r.kubecli, clusterName, namespace, ms, m, ec.Spec, owner, backupURL)
+	if err != nil {
+		return err
+	}
 	_, err = r.kubecli.CoreV1().Pods(ec.Namespace).Create(ctx, pod, metav1.CreateOptions{})
 	return err
 }

pkg/util/k8sutil/k8sutil.go

@@ -17,6 +17,7 @@ package k8sutil
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"net/url"
@@ -72,8 +73,14 @@ const (
 	// defaultDNSTimeout is the default maximum allowed time for the init container of the etcd pod
 	// to reverse DNS lookup its IP. The default behavior is to wait forever and has a value of 0.
 	defaultDNSTimeout = int64(0)
+
+	// discoveryEndpoint is the endpoint to be used for discovery service. The default is the public etcd
+	// service endpoint
+	discoveryEndpoint = "https://discovery.etcd.io"
 )
 
+var ErrDiscoveryTokenNotProvided = errors.New("cluster token not provided, you must provide a token when clustering mode is discovery")
+
 func GetEtcdVersion(pod *v1.Pod) string {
 	return pod.Annotations[etcdVersionAnnotationKey]
 }
@@ -306,10 +313,25 @@ func addOwnerRefToObject(o metav1.Object, r metav1.OwnerReference) {
 	o.SetOwnerReferences(append(o.GetOwnerReferences(), r))
 }
 
+func createToken(clusterSpec api.ClusterSpec) (string, error) {
+	if clusterSpec.ClusteringMode == "discovery" {
+		if clusterSpec.ClusterToken == "" {
+			return "", ErrDiscoveryTokenNotProvided
+		} else {
+			return clusterSpec.ClusterToken, nil
+		}
+	} else {
+		return uuid.New(), nil
+	}
+}
+
 // NewSeedMemberPod returns a Pod manifest for a seed member.
 // It's special that it has new token, and might need recovery init containers
 func NewSeedMemberPod(ctx context.Context, kubecli kubernetes.Interface, clusterName, clusterNamespace string, ms etcdutil.MemberSet, m *etcdutil.Member, cs api.ClusterSpec, owner metav1.OwnerReference, backupURL *url.URL) (*v1.Pod, error) {
-	token := uuid.New()
+	token, err := createToken(cs)
+	if err != nil {
+		return nil, err
+	}
 	pod, err := newEtcdPod(ctx, kubecli, m, ms.PeerURLPairs(), clusterName, clusterNamespace, "new", token, cs)
 	// TODO: PVC datadir support for restore process
 	AddEtcdVolumeToPod(pod, nil, cs.Pod.Tmpfs)
@@ -339,20 +361,39 @@ func ClientServiceName(clusterName string) string {
 	return clusterName + "-client"
 }
 
+func setupEtcdCommand(dataDir string, m *etcdutil.Member, initialCluster string, clusterState string, clusterToken string, clusteringMode string) (string, error) {
+	if clusteringMode == "discovery" {
+		command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+
+			"--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+
+			"--discovery=%s/%s",
+			dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), discoveryEndpoint, clusterToken)
+		return command, nil
+	} else {
+		command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+
+			"--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+
+			"--initial-cluster=%s --initial-cluster-state=%s",
+			dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), initialCluster, clusterState)
+		if clusterState == "new" {
+			command = fmt.Sprintf("%s --initial-cluster-token=%s", command, clusterToken)
+		}
+		return command, nil
+	}
+}
+
 func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.Member, initialCluster []string, clusterName, clusterNamespace, state, token string, cs api.ClusterSpec) (*v1.Pod, error) {
-	commands := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+
-		"--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+
-		"--initial-cluster=%s --initial-cluster-state=%s",
-		dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), strings.Join(initialCluster, ","), state)
+	command, err := setupEtcdCommand(dataDir, m, strings.Join(initialCluster, ","), state, token, cs.ClusteringMode)
+	if err != nil {
+		return nil, err
+	}
 	if m.SecurePeer {
 		secret, err := kubecli.CoreV1().Secrets(clusterNamespace).Get(ctx, cs.TLS.Static.Member.PeerSecret, metav1.GetOptions{})
 		if err != nil {
 			return nil, err
 		}
 		if secret.Type == v1.SecretTypeTLS {
-			commands += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/ca.crt --peer-cert-file=%[1]s/tls.crt --peer-key-file=%[1]s/tls.key", peerTLSDir)
+			command += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/ca.crt --peer-cert-file=%[1]s/tls.crt --peer-key-file=%[1]s/tls.key", peerTLSDir)
 		} else {
-			commands += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/peer-ca.crt --peer-cert-file=%[1]s/peer.crt --peer-key-file=%[1]s/peer.key", peerTLSDir)
+			command += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/peer-ca.crt --peer-cert-file=%[1]s/peer.crt --peer-key-file=%[1]s/peer.key", peerTLSDir)
 		}
 	}
 	if m.SecureClient {
@@ -361,14 +402,11 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M
 			return nil, err
 		}
 		if secret.Type == v1.SecretTypeTLS {
-			commands += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/ca.crt --cert-file=%[1]s/tls.crt --key-file=%[1]s/tls.key", serverTLSDir)
+			command += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/ca.crt --cert-file=%[1]s/tls.crt --key-file=%[1]s/tls.key", serverTLSDir)
 		} else {
-			commands += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/server-ca.crt --cert-file=%[1]s/server.crt --key-file=%[1]s/server.key", serverTLSDir)
+			command += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/server-ca.crt --cert-file=%[1]s/server.crt --key-file=%[1]s/server.key", serverTLSDir)
 		}
 	}
-	if state == "new" {
-		commands = fmt.Sprintf("%s --initial-cluster-token=%s", commands, token)
-	}
 
 	labels := map[string]string{
 		"app":          "etcd",
@@ -393,7 +431,7 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M
 	readinessProbe.FailureThreshold = 3
 
 	container := containerWithProbes(
-		etcdContainer(strings.Split(commands, " "), cs.Repository, cs.Version),
+		etcdContainer(strings.Split(command, " "), cs.Repository, cs.Version),
 		livenessProbe,
 		readinessProbe)