diff --git a/.github/workflows/go-build.yml b/.github/workflows/go-build.yml index eaff023a8..b908cb618 100644 --- a/.github/workflows/go-build.yml +++ b/.github/workflows/go-build.yml @@ -1,6 +1,5 @@ name: Build -on: - push: +on: [push, pull_request] jobs: operator: name: Operator diff --git a/.github/workflows/go-test-e2e.yml b/.github/workflows/go-test-e2e.yml index 5e80165e5..95202a216 100644 --- a/.github/workflows/go-test-e2e.yml +++ b/.github/workflows/go-test-e2e.yml @@ -1,6 +1,5 @@ name: Testing E2E -on: - push: +on: [push, pull_request] jobs: prepare: name: Prepare @@ -14,26 +13,41 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2.3.4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - - name: Docker login - run: docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" - env: - GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Prepare e2e image - run: | - hack/build/e2e/docker_push - env: - TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}} - + uses: docker/build-push-action@v4 + with: + push: true + context: . + file: test/pod/Dockerfile + tags: tfgco/etcd-operator-e2e:${{github.sha}} + - name: Prepare operator image run: | hack/build/operator/build hack/build/backup-operator/build hack/build/restore-operator/build - IMAGE=${OPERATOR_IMAGE} hack/build/docker_push - env: - OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}} + + - name: Prepare operator image + uses: docker/build-push-action@v4 + with: + push: true + context: . + file: hack/build/Dockerfile + tags: tfgco/etcd-operator:${{github.sha}} + test-e2e: name: E2E runs-on: ubuntu-latest @@ -48,12 +62,19 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2.3.4 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: KinD (Kubernetes in Docker) Action uses: engineerd/setup-kind@v0.5.0 - + with: + version: "v0.17.0" + - name: Test run: | - docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" docker pull $TEST_IMAGE docker pull $OPERATOR_IMAGE export KUBECONFIG="${HOME}/.kube/config" @@ -62,13 +83,13 @@ jobs: hack/ci/run_e2e env: GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} - OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}} + OPERATOR_IMAGE: tfgco/etcd-operator:${{github.sha}} TEST_AWS_SECRET: na TEST_S3_BUCKET: na TEST_NAMESPACE: default BUILD_IMAGE: false BUILD_E2E: false - TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}} + TEST_IMAGE: tfgco/etcd-operator-e2e:${{github.sha}} PASSES: e2e - name: Show logs diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 909fbba9d..a5c6939a2 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -1,6 +1,5 @@ name: Testing -on: - push: +on: [push, pull_request] jobs: build: diff --git a/example/rbac/cluster-role-binding-template.yaml b/example/rbac/cluster-role-binding-template.yaml index bee0ed128..483c6b0da 100644 --- a/example/rbac/cluster-role-binding-template.yaml +++ b/example/rbac/cluster-role-binding-template.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: diff --git a/example/rbac/cluster-role-template.yaml b/example/rbac/cluster-role-template.yaml index fa7a41e75..0a82e6a2a 100644 --- a/example/rbac/cluster-role-template.yaml +++ b/example/rbac/cluster-role-template.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: diff --git a/pkg/apis/etcd/v1beta2/cluster.go b/pkg/apis/etcd/v1beta2/cluster.go index 10125a700..24be6cd1a 100644 --- a/pkg/apis/etcd/v1beta2/cluster.go +++ b/pkg/apis/etcd/v1beta2/cluster.go @@ -102,6 +102,9 @@ type ClusterSpec struct { // etcd cluster TLS configuration TLS *TLSPolicy `json:"TLS,omitempty"` + + ClusteringMode string `json:"clusteringMode,omitempty"` + ClusterToken string `json:"clusterToken,omitempty"` } // PodPolicy defines the policy to create pod for the etcd container. diff --git a/pkg/controller/restore-operator/sync.go b/pkg/controller/restore-operator/sync.go index d985bf825..b6eef9450 100644 --- a/pkg/controller/restore-operator/sync.go +++ b/pkg/controller/restore-operator/sync.go @@ -225,6 +225,9 @@ func (r *Restore) createSeedMember(ctx context.Context, ec *api.EtcdCluster, svc backupURL := backupapi.BackupURLForRestore("http", svcAddr, clusterName, namespace) ec.SetDefaults() pod, err := k8sutil.NewSeedMemberPod(ctx, r.kubecli, clusterName, namespace, ms, m, ec.Spec, owner, backupURL) + if err != nil { + return err + } _, err = r.kubecli.CoreV1().Pods(ec.Namespace).Create(ctx, pod, metav1.CreateOptions{}) return err } diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 5ca1f0a39..4459ca717 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -17,6 +17,7 @@ package k8sutil import ( "context" "encoding/json" + "errors" "fmt" "net" "net/url" @@ -72,8 +73,14 @@ const ( // defaultDNSTimeout is the default maximum allowed time for the init container of the etcd pod // to reverse DNS lookup its IP. The default behavior is to wait forever and has a value of 0. defaultDNSTimeout = int64(0) + + // discoveryEndpoint is the endpoint to be used for discovery service. The default is the public etcd + // service endpoint + discoveryEndpoint = "https://discovery.etcd.io" ) +var ErrDiscoveryTokenNotProvided = errors.New("cluster token not provided, you must provide a token when clustering mode is discovery") + func GetEtcdVersion(pod *v1.Pod) string { return pod.Annotations[etcdVersionAnnotationKey] } @@ -306,10 +313,25 @@ func addOwnerRefToObject(o metav1.Object, r metav1.OwnerReference) { o.SetOwnerReferences(append(o.GetOwnerReferences(), r)) } +func createToken(clusterSpec api.ClusterSpec) (string, error) { + if clusterSpec.ClusteringMode == "discovery" { + if clusterSpec.ClusterToken == "" { + return "", ErrDiscoveryTokenNotProvided + } else { + return clusterSpec.ClusterToken, nil + } + } else { + return uuid.New(), nil + } +} + // NewSeedMemberPod returns a Pod manifest for a seed member. // It's special that it has new token, and might need recovery init containers func NewSeedMemberPod(ctx context.Context, kubecli kubernetes.Interface, clusterName, clusterNamespace string, ms etcdutil.MemberSet, m *etcdutil.Member, cs api.ClusterSpec, owner metav1.OwnerReference, backupURL *url.URL) (*v1.Pod, error) { - token := uuid.New() + token, err := createToken(cs) + if err != nil { + return nil, err + } pod, err := newEtcdPod(ctx, kubecli, m, ms.PeerURLPairs(), clusterName, clusterNamespace, "new", token, cs) // TODO: PVC datadir support for restore process AddEtcdVolumeToPod(pod, nil, cs.Pod.Tmpfs) @@ -339,20 +361,39 @@ func ClientServiceName(clusterName string) string { return clusterName + "-client" } +func setupEtcdCommand(dataDir string, m *etcdutil.Member, initialCluster string, clusterState string, clusterToken string, clusteringMode string) (string, error) { + if clusteringMode == "discovery" { + command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+ + "--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+ + "--discovery=%s/%s", + dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), discoveryEndpoint, clusterToken) + return command, nil + } else { + command := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+ + "--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+ + "--initial-cluster=%s --initial-cluster-state=%s", + dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), initialCluster, clusterState) + if clusterState == "new" { + command = fmt.Sprintf("%s --initial-cluster-token=%s", command, clusterToken) + } + return command, nil + } +} + func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.Member, initialCluster []string, clusterName, clusterNamespace, state, token string, cs api.ClusterSpec) (*v1.Pod, error) { - commands := fmt.Sprintf("/usr/local/bin/etcd --data-dir=%s --name=%s --initial-advertise-peer-urls=%s "+ - "--listen-peer-urls=%s --listen-client-urls=%s --advertise-client-urls=%s "+ - "--initial-cluster=%s --initial-cluster-state=%s", - dataDir, m.Name, m.PeerURL(), m.ListenPeerURL(), m.ListenClientURL(), m.ClientURL(), strings.Join(initialCluster, ","), state) + command, err := setupEtcdCommand(dataDir, m, strings.Join(initialCluster, ","), state, token, cs.ClusteringMode) + if err != nil { + return nil, err + } if m.SecurePeer { secret, err := kubecli.CoreV1().Secrets(clusterNamespace).Get(ctx, cs.TLS.Static.Member.PeerSecret, metav1.GetOptions{}) if err != nil { return nil, err } if secret.Type == v1.SecretTypeTLS { - commands += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/ca.crt --peer-cert-file=%[1]s/tls.crt --peer-key-file=%[1]s/tls.key", peerTLSDir) + command += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/ca.crt --peer-cert-file=%[1]s/tls.crt --peer-key-file=%[1]s/tls.key", peerTLSDir) } else { - commands += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/peer-ca.crt --peer-cert-file=%[1]s/peer.crt --peer-key-file=%[1]s/peer.key", peerTLSDir) + command += fmt.Sprintf(" --peer-client-cert-auth=true --peer-trusted-ca-file=%[1]s/peer-ca.crt --peer-cert-file=%[1]s/peer.crt --peer-key-file=%[1]s/peer.key", peerTLSDir) } } if m.SecureClient { @@ -361,14 +402,11 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M return nil, err } if secret.Type == v1.SecretTypeTLS { - commands += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/ca.crt --cert-file=%[1]s/tls.crt --key-file=%[1]s/tls.key", serverTLSDir) + command += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/ca.crt --cert-file=%[1]s/tls.crt --key-file=%[1]s/tls.key", serverTLSDir) } else { - commands += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/server-ca.crt --cert-file=%[1]s/server.crt --key-file=%[1]s/server.key", serverTLSDir) + command += fmt.Sprintf(" --client-cert-auth=true --trusted-ca-file=%[1]s/server-ca.crt --cert-file=%[1]s/server.crt --key-file=%[1]s/server.key", serverTLSDir) } } - if state == "new" { - commands = fmt.Sprintf("%s --initial-cluster-token=%s", commands, token) - } labels := map[string]string{ "app": "etcd", @@ -393,7 +431,7 @@ func newEtcdPod(ctx context.Context, kubecli kubernetes.Interface, m *etcdutil.M readinessProbe.FailureThreshold = 3 container := containerWithProbes( - etcdContainer(strings.Split(commands, " "), cs.Repository, cs.Version), + etcdContainer(strings.Split(command, " "), cs.Repository, cs.Version), livenessProbe, readinessProbe) diff --git a/pkg/util/k8sutil/k8sutils_test.go b/pkg/util/k8sutil/k8sutils_test.go index cce491401..1eacf7296 100644 --- a/pkg/util/k8sutil/k8sutils_test.go +++ b/pkg/util/k8sutil/k8sutils_test.go @@ -15,9 +15,11 @@ package k8sutil import ( + "strings" "testing" api "github.com/coreos/etcd-operator/pkg/apis/etcd/v1beta2" + "github.com/coreos/etcd-operator/pkg/util/etcdutil" ) func TestDefaultBusyboxImageName(t *testing.T) { @@ -47,3 +49,170 @@ func TestSetBusyboxImageName(t *testing.T) { t.Errorf("expect image=%s, get=%s", expected, image) } } + +func TestEtcdCommandNewLocalCluster(t *testing.T) { + dataDir := "/var/etcd/data" + etcdMember := &etcdutil.Member{ + Name: "etcd-test", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + memberSet := etcdutil.NewMemberSet(etcdMember).PeerURLPairs() + clusterState := "new" + token := "token" + + initialEtcdCommand, _ := setupEtcdCommand(dataDir, etcdMember, strings.Join(memberSet, ","), clusterState, token, "") + + expectedCommand := "/usr/local/bin/etcd --data-dir=/var/etcd/data --name=etcd-test --initial-advertise-peer-urls=http://etcd-test.etcd.etcd.svc.local:2380 " + + "--listen-peer-urls=http://0.0.0.0:2380 --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://etcd-test.etcd.etcd.svc.local:2379 " + + "--initial-cluster=etcd-test=http://etcd-test.etcd.etcd.svc.local:2380 --initial-cluster-state=new --initial-cluster-token=token" + + if initialEtcdCommand != expectedCommand { + t.Errorf("expected command=%s, got=%s", expectedCommand, initialEtcdCommand) + } +} + +//TODO +func TestEtcdCommandExistingLocalCluster(t *testing.T) { + dataDir := "/var/etcd/data" + etcdMember1 := &etcdutil.Member{ + Name: "etcd-test-1", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + etcdMember2 := &etcdutil.Member{ + Name: "etcd-test-2", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + memberSet := etcdutil.NewMemberSet(etcdMember1) + memberSet.Add(etcdMember2) + memberSetURLs := memberSet.PeerURLPairs() + token := "token" + clusterState := "existing" + + initialEtcdCommand, _ := setupEtcdCommand(dataDir, etcdMember2, strings.Join(memberSetURLs, ","), clusterState, token, "") + + commandBeforeClusterSet := "/usr/local/bin/etcd --data-dir=/var/etcd/data --name=etcd-test-2 --initial-advertise-peer-urls=http://etcd-test-2.etcd-test.etcd.svc.local:2380 " + + "--listen-peer-urls=http://0.0.0.0:2380 --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://etcd-test-2.etcd-test.etcd.svc.local:2379 " + commandClusterSet1 := "--initial-cluster=etcd-test-1=http://etcd-test-1.etcd-test.etcd.svc.local:2380,etcd-test-2=http://etcd-test-2.etcd-test.etcd.svc.local:2380 --initial-cluster-state=existing" + commandClusterSet2 := "--initial-cluster=etcd-test-2=http://etcd-test-2.etcd-test.etcd.svc.local:2380,etcd-test-1=http://etcd-test-1.etcd-test.etcd.svc.local:2380 --initial-cluster-state=existing" + expectedCommand1 := commandBeforeClusterSet+commandClusterSet1 + expectedCommand2 := commandBeforeClusterSet+commandClusterSet2 + + if initialEtcdCommand != expectedCommand1 && initialEtcdCommand != expectedCommand2{ + t.Errorf("wrong etcd command, got=%s", initialEtcdCommand) + } +} + +//ToDo +func TestEtcdCommandInvalidClusterMode(t *testing.T) { + dataDir := "/var/etcd/data" + etcdMember := &etcdutil.Member{ + Name: "etcd-test", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + memberSet := etcdutil.NewMemberSet(etcdMember).PeerURLPairs() + clusterState := "new" + token := "token" + clusteringMode := "invalid" + + initialEtcdCommand, _ := setupEtcdCommand(dataDir, etcdMember, strings.Join(memberSet, ","), clusterState, token, clusteringMode) + + expectedCommand := "/usr/local/bin/etcd --data-dir=/var/etcd/data --name=etcd-test --initial-advertise-peer-urls=http://etcd-test.etcd.etcd.svc.local:2380 " + + "--listen-peer-urls=http://0.0.0.0:2380 --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://etcd-test.etcd.etcd.svc.local:2379 " + + "--initial-cluster=etcd-test=http://etcd-test.etcd.etcd.svc.local:2380 --initial-cluster-state=new --initial-cluster-token=token" + + if initialEtcdCommand != expectedCommand { + t.Errorf("expected command=%s, got=%s", expectedCommand, initialEtcdCommand) + } +} + +func TestEtcdCommandDiscoveryCluster(t *testing.T) { + dataDir := "/var/etcd/data" + etcdMember := &etcdutil.Member{ + Name: "etcd-test", + Namespace: "etcd", + SecurePeer: false, + SecureClient: false, + ClusterDomain: ".local", + } + memberSet := etcdutil.NewMemberSet(etcdMember).PeerURLPairs() + clusterState := "new" + clusterToken := "token" + clusteringMode := "discovery" + + initialEtcdCommand, _ := setupEtcdCommand(dataDir, etcdMember, strings.Join(memberSet, ","), clusterState, clusterToken, clusteringMode) + + expectedCommand := "/usr/local/bin/etcd --data-dir=/var/etcd/data --name=etcd-test --initial-advertise-peer-urls=http://etcd-test.etcd.etcd.svc.local:2380 " + + "--listen-peer-urls=http://0.0.0.0:2380 --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://etcd-test.etcd.etcd.svc.local:2379 " + + "--discovery=https://discovery.etcd.io/token" + + if initialEtcdCommand != expectedCommand { + t.Errorf("expected command=%s, got=%s", expectedCommand, initialEtcdCommand) + } +} + +func TestCreateTokenLocalCluster(t *testing.T) { + clusterSpec := &api.ClusterSpec{ + Size: 1, + ClusteringMode: "local", + ClusterToken: "testtoken", + } + + token, _ := createToken(*clusterSpec) + + if token == "testtoken" { + t.Errorf("token should be a randon uuid, instead got %s", token) + } +} + +func TestCreateTokenDiscoveryClusterNoTokenSent(t *testing.T) { + clusterSpec := &api.ClusterSpec{ + Size: 1, + ClusteringMode: "discovery", + } + + _, err := createToken(*clusterSpec) + + if err == nil { + t.Errorf("Expected an error to be thrown when discovery mode on and no token is set") + } +} + +func TestCreateTokenDiscoveryClusterTokenEmpty(t *testing.T) { + clusterSpec := &api.ClusterSpec{ + Size: 1, + ClusteringMode: "discovery", + ClusterToken: "", + } + + _, err := createToken(*clusterSpec) + + if err == nil { + t.Errorf("Expected an error to be thrown when discovery mode on and no token is set") + } +} + +func TestCreateTokenDistributedCluster(t *testing.T) { + clusterSpec := &api.ClusterSpec{ + Size: 1, + ClusteringMode: "discovery", + ClusterToken: "testtoken", + } + + token, _ := createToken(*clusterSpec) + + if token != "testtoken" { + t.Errorf("expected token=%s, got=%s", clusterSpec.ClusterToken, token) + } +} diff --git a/pkg/util/k8sutil/service_utils_test.go b/pkg/util/k8sutil/service_utils_test.go index 426549152..3b452bdee 100644 --- a/pkg/util/k8sutil/service_utils_test.go +++ b/pkg/util/k8sutil/service_utils_test.go @@ -165,7 +165,7 @@ func TestApplyServicePolicyWithClusterIP(t *testing.T) { } applyServicePolicy(svc, policy) actualType := svc.Spec.Type - actualClusterIP := svc.Spec.Type + actualClusterIP := svc.Spec.ClusterIP if !reflect.DeepEqual(serviceType, actualType) { t.Errorf("expect expected=%v, got=%v", serviceType, actualType) }