fix

Makefile

@@ -82,7 +82,7 @@ test: manifests generate check ## Run tests.
 
 ##@ e2e
 e2e: $(GOBIN)/ginkgo $(GOBIN)/kind helm
-	./hack/e2e.sh
+	PATH="${GOBIN}:${PATH}" ./hack/e2e.sh
 
 ##@ Build
 

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-logr/logr v0.3.0
 	github.com/google/go-cmp v0.5.2
+	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/openkruise/kruise-api v0.8.0
 	github.com/pkg/errors v0.9.1

hack/e2e.sh

@@ -16,28 +16,40 @@
 
 set -euo pipefail
 
-ROOT=$(cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 
 INSTALL_KUBERNETES=${INSTALL_KUBERNETES:-true}
 UNINSTALL_KUBERNETES=${UNINSTALL_KUBERNETES:-true}
 INSTALL_CERT_MANAGER=${INSTALL_CERT_MANAGER:-true}
+INSTALL_CERT_MANAGER_VERSION=${INSTALL_CERT_MANAGER_VERSION:-v1.3.1}
 INSTALL_KRUISE=${INSTALL_KRUISE:-true}
+INSTALL_KRUISE_VERSION=${INSTALL_KRUISE_VERSION:-v0.8.1}
 INSTALL_NEBULA_OPERATOR=${INSTALL_NEBULA_OPERATOR:-true}
 KIND_NAME=${KIND_NAME:-e2e-test}
 KIND_CONFIG=${KIND_CONFIG:-${ROOT}/hack/kind-config.yaml}
+STORAGE_CLASS=${STORAGE_CLASS:-}
+NEBULA_VERSION=${NEBULA_VERSION:-v2-nightly}
 
-KUBECONFIG=${KUBECONFIG:-~/.kube/${KIND_NAME}.kind.config}
+if [[ "${INSTALL_KUBERNETES}" == "true" ]];then
+  KUBECONFIG=~/.kube/${KIND_NAME}.kind.config
+else
+  KUBECONFIG=${KUBECONFIG:-~/.kube/config}
+fi
 DELETE_NAMESPACE=${DELETE_NAMESPACE:-true}
 DELETE_NAMESPACE_ON_FAILURE=${DELETE_NAMESPACE_ON_FAILURE:-false}
 
 echo "starting e2e tests"
 echo "INSTALL_KUBERNETES: ${INSTALL_KUBERNETES}"
 echo "UNINSTALL_KUBERNETES: ${UNINSTALL_KUBERNETES}"
 echo "INSTALL_CERT_MANAGER: ${INSTALL_CERT_MANAGER}"
+echo "INSTALL_CERT_MANAGER_VERSION: ${INSTALL_CERT_MANAGER_VERSION}"
 echo "INSTALL_KRUISE: ${INSTALL_KRUISE}"
+echo "INSTALL_KRUISE_VERSION: ${INSTALL_KRUISE_VERSION}"
 echo "INSTALL_NEBULA_OPERATOR: ${INSTALL_NEBULA_OPERATOR}"
 echo "KIND_NAME: ${KIND_NAME}"
 echo "KIND_CONFIG: ${KIND_CONFIG}"
+echo "STORAGE_CLASS: ${STORAGE_CLASS}"
+echo "NEBULA_VERSION: ${NEBULA_VERSION}"
 
 echo "KUBECONFIG: ${KUBECONFIG}"
 echo "DELETE_NAMESPACE: ${DELETE_NAMESPACE}"
@@ -46,14 +58,18 @@ echo "DELETE_NAMESPACE_ON_FAILURE: ${DELETE_NAMESPACE_ON_FAILURE}"
 
 ginkgo ./tests/e2e \
   -- \
-  --install-kubernetes=${INSTALL_KUBERNETES} \
-  --uninstall-kubernetes=${UNINSTALL_KUBERNETES} \
-  --install-cert-manager=${INSTALL_CERT_MANAGER} \
-  --install-kruise=${INSTALL_KRUISE} \
-  --install-nebula-operator=${INSTALL_NEBULA_OPERATOR} \
-  --kind-name=${KIND_NAME} \
-  --kind-config=${KIND_CONFIG} \
-  --kubeconfig=${KUBECONFIG} \
-  --delete-namespace=${DELETE_NAMESPACE} \
-  --delete-namespace-on-failure=${DELETE_NAMESPACE_ON_FAILURE} \
+  --install-kubernetes="${INSTALL_KUBERNETES}" \
+  --uninstall-kubernetes="${UNINSTALL_KUBERNETES}" \
+  --install-cert-manager="${INSTALL_CERT_MANAGER}" \
+  --install-cert-manager-version="${INSTALL_CERT_MANAGER_VERSION}" \
+  --install-kruise="${INSTALL_KRUISE}" \
+  --install-kruise-version="${INSTALL_KRUISE_VERSION}" \
+  --install-nebula-operator="${INSTALL_NEBULA_OPERATOR}" \
+  --kind-name="${KIND_NAME}" \
+  --kind-config="${KIND_CONFIG}" \
+  --storage-class="${STORAGE_CLASS}" \
+  --nebula-version="${NEBULA_VERSION}" \
+  --kubeconfig="${KUBECONFIG}" \
+  --delete-namespace="${DELETE_NAMESPACE}" \
+  --delete-namespace-on-failure="${DELETE_NAMESPACE_ON_FAILURE}" \
   "${@}"

tests/e2e/config/config.go

@@ -21,19 +21,25 @@ import (
 )
 
 const (
-	DefaultKindName = "e2e-test"
+	DefaultKindName                  = "e2e-test"
+	DefaultInstallCertManagerVersion = "v1.3.1"
+	DefaultInstallKruiseVersion      = "v0.8.1"
 )
 
 var TestConfig Config
 
 type Config struct {
-	InstallKubernetes     bool
-	UninstallKubernetes   bool
-	InstallCertManager    bool
-	InstallKruise         bool
-	InstallNebulaOperator bool
-	KindName              string
-	KindConfig            string
+	InstallKubernetes         bool
+	UninstallKubernetes       bool
+	InstallCertManager        bool
+	InstallCertManagerVersion string
+	InstallKruise             bool
+	InstallKruiseVersion      string
+	InstallNebulaOperator     bool
+	KindName                  string
+	KindConfig                string
+	StorageClass              string
+	NebulaVersion             string
 }
 
 func RegisterClusterFlags(flags *flag.FlagSet) {
@@ -43,12 +49,21 @@ func RegisterClusterFlags(flags *flag.FlagSet) {
 		"If true tests will uninstall kubernetes. Ignored when --install-kubernetes is false.")
 	flags.BoolVar(&TestConfig.InstallCertManager, "install-cert-manager", true,
 		"If true tests will install cert-manager.")
+	flags.StringVar(&TestConfig.InstallCertManagerVersion, "install-cert-manager-version", DefaultInstallCertManagerVersion,
+		"The cert-manager version to install.")
 	flags.BoolVar(&TestConfig.InstallKruise, "install-kruise", true,
 		"If true tests will install kruise.")
+	flags.StringVar(&TestConfig.InstallKruiseVersion, "install-kruise-version", DefaultInstallKruiseVersion,
+		"The kruise version to install.")
 	flags.BoolVar(&TestConfig.InstallNebulaOperator, "install-nebula-operator", true,
 		"If true tests will install nebula operator.")
 	flags.StringVar(&TestConfig.KindName, "kind-name", DefaultKindName,
 		"The kind name to install.")
 	flags.StringVar(&TestConfig.KindConfig, "kind-config", "../../hack/kind-config.yaml",
 		"The kind config to install.")
+	flags.StringVar(&TestConfig.StorageClass, "storage-class", "",
+		"The storage class to use to install nebula cluster."+
+			"If don't configure, use the default storage class and then the others in the kubernetes.")
+	flags.StringVar(&TestConfig.NebulaVersion, "nebula-version", "v2-nightly",
+		"The nebula version.")
 }

tests/e2e/nebulacluster/nebulacluster.go

@@ -104,7 +104,7 @@ var _ = ginkgo.Describe("NebulaCluster", func() {
 
 					var err error
 					// init the NebulaCluster Resource for testing
-					nc := getNebulaCluster(ns, tc.Name)
+					nc := getNebulaCluster(runtimeClient, ns, tc.Name)
 					nc.Spec.Graphd.Replicas = pointer.Int32Ptr(tc.GraphdReplicas)
 					nc.Spec.Metad.Replicas = pointer.Int32Ptr(tc.MetadReplicas)
 					nc.Spec.Storaged.Replicas = pointer.Int32Ptr(tc.StoragedReplicas)
@@ -183,7 +183,7 @@ var _ = ginkgo.Describe("NebulaCluster", func() {
 		ginkgo.By("Deploy NebulaCluster")
 		var err error
 
-		nc := getNebulaCluster(ns, "test-recovery")
+		nc := getNebulaCluster(runtimeClient, ns, "test-recovery")
 		nc.Spec.Graphd.Replicas = pointer.Int32Ptr(2)
 		nc.Spec.Metad.Replicas = pointer.Int32Ptr(3)
 		nc.Spec.Storaged.Replicas = pointer.Int32Ptr(4)

tests/e2e/nebulacluster/util.go

@@ -25,25 +25,29 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
+	storageutil "k8s.io/kubernetes/pkg/apis/storage/util"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	nebula "github.com/vesoft-inc/nebula-go"
 	"github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1"
 	"github.com/vesoft-inc/nebula-operator/pkg/label"
+	e2econfig "github.com/vesoft-inc/nebula-operator/tests/e2e/config"
 	e2eutil "github.com/vesoft-inc/nebula-operator/tests/e2e/util"
 )
 
-func getNebulaCluster(namespace, name string) *v1alpha1.NebulaCluster {
+func getNebulaCluster(runtimeClient client.Client, namespace, name string) *v1alpha1.NebulaCluster {
 	imagePullPolicy := corev1.PullIfNotPresent
-	storageClassName := "standard" // standard, fast-disks
+	storageClassName := getStorageClassName(runtimeClient)
+	nebulaVersion := e2econfig.TestConfig.NebulaVersion
 	return &v1alpha1.NebulaCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -54,7 +58,7 @@ func getNebulaCluster(namespace, name string) *v1alpha1.NebulaCluster {
 				PodSpec: v1alpha1.PodSpec{
 					Replicas: pointer.Int32Ptr(1),
 					Image:    "vesoft/nebula-graphd",
-					Version:  "v2-nightly",
+					Version:  nebulaVersion,
 				},
 				StorageClaim: &v1alpha1.StorageClaim{
 					StorageClassName: &storageClassName,
@@ -69,7 +73,7 @@ func getNebulaCluster(namespace, name string) *v1alpha1.NebulaCluster {
 				PodSpec: v1alpha1.PodSpec{
 					Replicas: pointer.Int32Ptr(1),
 					Image:    "vesoft/nebula-metad",
-					Version:  "v2-nightly",
+					Version:  nebulaVersion,
 				},
 				StorageClaim: &v1alpha1.StorageClaim{
 					StorageClassName: &storageClassName,
@@ -84,7 +88,7 @@ func getNebulaCluster(namespace, name string) *v1alpha1.NebulaCluster {
 				PodSpec: v1alpha1.PodSpec{
 					Replicas: pointer.Int32Ptr(1),
 					Image:    "vesoft/nebula-storaged",
-					Version:  "v2-nightly",
+					Version:  nebulaVersion,
 				},
 				StorageClaim: &v1alpha1.StorageClaim{
 					StorageClassName: &storageClassName,
@@ -105,6 +109,28 @@ func getNebulaCluster(namespace, name string) *v1alpha1.NebulaCluster {
 	}
 }
 
+func getStorageClassName(runtimeClient client.Client) string {
+	if e2econfig.TestConfig.StorageClass != "" {
+		return e2econfig.TestConfig.StorageClass
+	}
+
+	var scList storagev1.StorageClassList
+	err := runtimeClient.List(context.TODO(), &scList)
+	framework.ExpectNoError(err, "failed to list StorageClass")
+	framework.ExpectNotEqual(len(scList.Items), 0, "don't find StorageClass")
+	var scName string
+	for i := range scList.Items {
+		sc := scList.Items[i]
+		if storageutil.IsDefaultAnnotation(sc.ObjectMeta) {
+			return sc.GetName()
+		}
+		if scName == "" {
+			scName = sc.GetName()
+		}
+	}
+	return scName
+}
+
 func waitForNebulaClusterReady(
 	nc *v1alpha1.NebulaCluster,
 	timeout, pollInterval time.Duration,

tests/e2e/suites.go

@@ -129,7 +129,7 @@ func setupCertManager() {
 		"--repo", "https://charts.jetstack.io",
 		"--namespace", helmNamespace,
 		"--create-namespace",
-		"--version", "v1.3.1",
+		"--version", e2econfig.TestConfig.InstallCertManagerVersion,
 		"--set", "installCRDs=true",
 	}
 	helmInstall(helmName, helmNamespace, workloadNamespaces, helmArgs...)
@@ -144,7 +144,7 @@ func setupKruise() {
 	workloadNamespaces := []string{"kruise-system"}
 	helmArgs := []string{
 		"install", helmName,
-		"https://github.com/openkruise/kruise/releases/download/v0.8.1/kruise-chart.tgz",
+		fmt.Sprintf("https://github.com/openkruise/kruise/releases/download/%s/kruise-chart.tgz", e2econfig.TestConfig.InstallKruiseVersion),
 	}
 	helmInstall(helmName, helmNamespace, workloadNamespaces, helmArgs...)
 }