add e2e tests

Makefile

@@ -78,7 +78,11 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test: manifests generate check ## Run tests.
 	mkdir -p ${ENVTEST_ASSETS_DIR}
 	test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.0/hack/setup-envtest.sh
-	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
+	source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./pkg/... -coverprofile cover.out
+
+##@ e2e
+e2e: $(GOBIN)/ginkgo $(GOBIN)/kind helm
+	PATH="${GOBIN}:${PATH}" ./hack/e2e.sh
 
 ##@ Build
 
@@ -122,7 +126,10 @@ tools: $(GOBIN)/goimports \
 	$(GOBIN)/gofumpt \
 	$(GOBIN)/golangci-lint \
 	$(GOBIN)/controller-gen \
-	$(GOBIN)/kustomize
+	$(GOBIN)/kustomize \
+	$(GOBIN)/ginkgo \
+	$(GOBIN)/kind \
+	helm
 
 $(GOBIN)/goimports:
 	$(call go-get-tool,$(GOBIN)/goimports,golang.org/x/tools/cmd/goimports)
@@ -148,6 +155,18 @@ $(GOBIN)/controller-gen:
 $(GOBIN)/kustomize:
 	$(call go-get-tool,$(GOBIN)/kustomize,sigs.k8s.io/kustomize/kustomize/[email protected])
 
+$(GOBIN)/ginkgo:
+	$(call go-get-tool,$(GOBIN)/kustomize,github.com/onsi/ginkgo/[email protected])
+
+$(GOBIN)/kind:
+	$(call go-get-tool,$(GOBIN)/kustomize,sigs.k8s.io/[email protected])
+
+helm:
+	@[ -f /usr/local/bin/helm ] || { \
+	set -e ;\
+	curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash ;\
+	}
+
 # go-get-tool will 'go get' any package $2 and install it to $1.
 define go-get-tool
 @[ -f $(1) ] || { \

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-logr/logr v0.3.0
 	github.com/google/go-cmp v0.5.2
+	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/openkruise/kruise-api v0.8.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -146,11 +146,13 @@ github.com/docker/docker v1.4.2-0.20200309214505-aa6a9891b09c/go.mod h1:eEKB0N0r
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
@@ -833,6 +835,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
@@ -891,6 +894,7 @@ k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H
 k8s.io/kube-proxy v0.19.2/go.mod h1:/07XChnL0EkYAQyZ7noQuyPYX5QOUBQECa9dsm9ScyY=
 k8s.io/kube-scheduler v0.19.2 h1:xV2Yj76g62n4+g6IR7/8Nba5VfCI53CCrD6qnQ4yaH0=
 k8s.io/kube-scheduler v0.19.2/go.mod h1:Mh/QNfmP0eqt7JtNUyIAsGhU2zO4j1EWel8TFizxZKo=
+k8s.io/kubectl v0.19.2 h1:/Dxz9u7S0GnchLA6Avqi5k1qhZH4Fusgecj8dHsSnbk=
 k8s.io/kubectl v0.19.2/go.mod h1:4ib3oj5ma6gF95QukTvC7ZBMxp60+UEAhDPjLuBIrV4=
 k8s.io/kubelet v0.19.2/go.mod h1:FHHoByVWzh6kNaarXaDPAa751Oz6REcOVRyFT84L1Is=
 k8s.io/kubernetes v1.19.2 h1:sEvBYVM1/H5hqejFR10u8ndreYARV3DiTrqi2AY31ok=

hack/e2e.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 Vesoft Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+
+INSTALL_KUBERNETES=${INSTALL_KUBERNETES:-true}
+UNINSTALL_KUBERNETES=${UNINSTALL_KUBERNETES:-true}
+INSTALL_CERT_MANAGER=${INSTALL_CERT_MANAGER:-true}
+INSTALL_CERT_MANAGER_VERSION=${INSTALL_CERT_MANAGER_VERSION:-v1.3.1}
+INSTALL_KRUISE=${INSTALL_KRUISE:-true}
+INSTALL_KRUISE_VERSION=${INSTALL_KRUISE_VERSION:-v0.8.1}
+INSTALL_NEBULA_OPERATOR=${INSTALL_NEBULA_OPERATOR:-true}
+KIND_NAME=${KIND_NAME:-e2e-test}
+KIND_CONFIG=${KIND_CONFIG:-${ROOT}/hack/kind-config.yaml}
+STORAGE_CLASS=${STORAGE_CLASS:-}
+NEBULA_VERSION=${NEBULA_VERSION:-v2-nightly}
+
+if [[ "${INSTALL_KUBERNETES}" == "true" ]];then
+  KUBECONFIG=~/.kube/${KIND_NAME}.kind.config
+else
+  KUBECONFIG=${KUBECONFIG:-~/.kube/config}
+fi
+DELETE_NAMESPACE=${DELETE_NAMESPACE:-true}
+DELETE_NAMESPACE_ON_FAILURE=${DELETE_NAMESPACE_ON_FAILURE:-false}
+
+echo "starting e2e tests"
+echo "INSTALL_KUBERNETES: ${INSTALL_KUBERNETES}"
+echo "UNINSTALL_KUBERNETES: ${UNINSTALL_KUBERNETES}"
+echo "INSTALL_CERT_MANAGER: ${INSTALL_CERT_MANAGER}"
+echo "INSTALL_CERT_MANAGER_VERSION: ${INSTALL_CERT_MANAGER_VERSION}"
+echo "INSTALL_KRUISE: ${INSTALL_KRUISE}"
+echo "INSTALL_KRUISE_VERSION: ${INSTALL_KRUISE_VERSION}"
+echo "INSTALL_NEBULA_OPERATOR: ${INSTALL_NEBULA_OPERATOR}"
+echo "KIND_NAME: ${KIND_NAME}"
+echo "KIND_CONFIG: ${KIND_CONFIG}"
+echo "STORAGE_CLASS: ${STORAGE_CLASS}"
+echo "NEBULA_VERSION: ${NEBULA_VERSION}"
+
+echo "KUBECONFIG: ${KUBECONFIG}"
+echo "DELETE_NAMESPACE: ${DELETE_NAMESPACE}"
+echo "INSTALL_KUBERNETES: ${INSTALL_KUBERNETES}"
+echo "DELETE_NAMESPACE_ON_FAILURE: ${DELETE_NAMESPACE_ON_FAILURE}"
+
+ginkgo ./tests/e2e \
+  -- \
+  --install-kubernetes="${INSTALL_KUBERNETES}" \
+  --uninstall-kubernetes="${UNINSTALL_KUBERNETES}" \
+  --install-cert-manager="${INSTALL_CERT_MANAGER}" \
+  --install-cert-manager-version="${INSTALL_CERT_MANAGER_VERSION}" \
+  --install-kruise="${INSTALL_KRUISE}" \
+  --install-kruise-version="${INSTALL_KRUISE_VERSION}" \
+  --install-nebula-operator="${INSTALL_NEBULA_OPERATOR}" \
+  --kind-name="${KIND_NAME}" \
+  --kind-config="${KIND_CONFIG}" \
+  --storage-class="${STORAGE_CLASS}" \
+  --nebula-version="${NEBULA_VERSION}" \
+  --kubeconfig="${KUBECONFIG}" \
+  --delete-namespace="${DELETE_NAMESPACE}" \
+  --delete-namespace-on-failure="${DELETE_NAMESPACE_ON_FAILURE}" \
+  "${@}"

hack/kind-config.yaml

@@ -0,0 +1,22 @@
+# Copyright 2021 Vesoft Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker
+- role: worker
+- role: worker

hack/update-codegen.sh

@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
 
+# Copyright 2021 Vesoft Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -o errexit
 set -o nounset
 set -o pipefail

tests/e2e/config/config.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2021 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"flag"
+)
+
+const (
+	DefaultKindName                  = "e2e-test"
+	DefaultInstallCertManagerVersion = "v1.3.1"
+	DefaultInstallKruiseVersion      = "v0.8.1"
+)
+
+var TestConfig Config
+
+type Config struct {
+	InstallKubernetes         bool
+	UninstallKubernetes       bool
+	InstallCertManager        bool
+	InstallCertManagerVersion string
+	InstallKruise             bool
+	InstallKruiseVersion      string
+	InstallNebulaOperator     bool
+	KindName                  string
+	KindConfig                string
+	StorageClass              string
+	NebulaVersion             string
+}
+
+func RegisterClusterFlags(flags *flag.FlagSet) {
+	flags.BoolVar(&TestConfig.InstallKubernetes, "install-kubernetes", true,
+		"If true tests will install kubernetes.")
+	flags.BoolVar(&TestConfig.UninstallKubernetes, "uninstall-kubernetes", true,
+		"If true tests will uninstall kubernetes. Ignored when --install-kubernetes is false.")
+	flags.BoolVar(&TestConfig.InstallCertManager, "install-cert-manager", true,
+		"If true tests will install cert-manager.")
+	flags.StringVar(&TestConfig.InstallCertManagerVersion, "install-cert-manager-version", DefaultInstallCertManagerVersion,
+		"The cert-manager version to install.")
+	flags.BoolVar(&TestConfig.InstallKruise, "install-kruise", true,
+		"If true tests will install kruise.")
+	flags.StringVar(&TestConfig.InstallKruiseVersion, "install-kruise-version", DefaultInstallKruiseVersion,
+		"The kruise version to install.")
+	flags.BoolVar(&TestConfig.InstallNebulaOperator, "install-nebula-operator", true,
+		"If true tests will install nebula operator.")
+	flags.StringVar(&TestConfig.KindName, "kind-name", DefaultKindName,
+		"The kind name to install.")
+	flags.StringVar(&TestConfig.KindConfig, "kind-config", "../../hack/kind-config.yaml",
+		"The kind config to install.")
+	flags.StringVar(&TestConfig.StorageClass, "storage-class", "",
+		"The storage class to use to install nebula cluster."+
+			"If don't configure, use the default storage class and then the others in the kubernetes.")
+	flags.StringVar(&TestConfig.NebulaVersion, "nebula-version", "v2-nightly",
+		"The nebula version.")
+}

tests/e2e/e2e.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2021 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	setupSuite()
+	return nil
+}, func(_ []byte) {
+	setupSuitePerGinkgoNode()
+})
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	cleanupSuitePerGinkgoNode()
+}, func() {
+	cleanupSuite()
+})
+
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "nebula-operator e2e suite")
+}

tests/e2e/e2e_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2021 Vesoft Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"flag"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/config"
+
+	e2econfig "github.com/vesoft-inc/nebula-operator/tests/e2e/config"
+	_ "github.com/vesoft-inc/nebula-operator/tests/e2e/nebulacluster"
+)
+
+func handleFlags() {
+	config.CopyFlags(config.Flags, flag.CommandLine)
+	framework.RegisterCommonFlags(flag.CommandLine)
+	framework.RegisterClusterFlags(flag.CommandLine)
+	e2econfig.RegisterClusterFlags(flag.CommandLine)
+	flag.Parse()
+}
+
+func TestMain(m *testing.M) {
+	handleFlags()
+
+	flag.CommandLine.VisitAll(func(flag *flag.Flag) {
+		framework.Logf("FLAG: --%s=%q", flag.Name, flag.Value)
+	})
+
+	framework.AfterReadingAllFlags(&framework.TestContext)
+
+	rand.Seed(time.Now().UnixNano())
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}