feat: add resources e2e (#329)

* feat: add resources e2e

* fix: lint

* feat: adjust

* fix: lint

* fix: comments

* fix

tests/e2e/e2evalidator/e2evalidator.go

@@ -8,9 +8,28 @@ import (
 	"github.com/go-playground/validator/v10"
 )
 
+type Ruler interface {
+	Cmp(val any) error
+}
+
 type Rule string
 
-func StructWithRules(actual any, rulesMapping map[string]Rule) (errMessages []string) {
+func (r Rule) Cmp(val any) error {
+	return validator.New().Var(val, string(r))
+}
+
+type AnyStruct struct {
+	Val any
+}
+
+func (a AnyStruct) Cmp(val any) error {
+	if !reflect.DeepEqual(val, a.Val) {
+		return fmt.Errorf("objects not equal, expected %v, got %v", a, val)
+	}
+	return nil
+}
+
+func StructWithRules(actual any, rulesMapping map[string]Ruler) (errMessages []string) {
 	for field, rule := range rulesMapping {
 		actualVal := reflect.ValueOf(actual)
 
@@ -32,7 +51,8 @@ func StructWithRules(actual any, rulesMapping map[string]Rule) (errMessages []st
 				actualVal = actualVal.Elem()
 			}
 			val := actualVal.Interface()
-			if err := validator.New().Var(val, string(rule)); err != nil {
+
+			if err := rule.Cmp(val); err != nil {
 				errMessages = append(errMessages, fmt.Sprintf("field %q(%v) does not match %q\n", field, val, rule))
 			}
 		}

tests/e2e/envfuncsext/nebulacluster-ready-func.go

@@ -19,7 +19,10 @@ package envfuncsext
 import (
 	"context"
 	stderrors "errors"
+	"reflect"
 
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 
@@ -46,7 +49,7 @@ func DefaultNebulaClusterReadyFunc(ctx context.Context, cfg *envconf.Config, nc
 	return true, nil
 }
 
-func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping map[string]e2evalidator.Rule) NebulaClusterReadyFunc {
+func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping map[string]e2evalidator.Ruler) NebulaClusterReadyFunc {
 	return func(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (isReady bool, err error) {
 		if errMessages := e2evalidator.StructWithRules(nc, rulesMapping); len(errMessages) > 0 {
 			if ignoreValidationErrors {
@@ -61,7 +64,7 @@ func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping m
 	}
 }
 
-func defaultNebulaClusterReadyFuncForStatus(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
+func defaultNebulaClusterReadyFuncForStatus(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
 	isReady := nc.IsReady()
 	if isReady {
 		// TODO: Add more checks
@@ -115,19 +118,40 @@ func defaultNebulaClusterReadyFuncForStatus(_ context.Context, _ *envconf.Config
 	return isReady, nil
 }
 
-func defaultNebulaClusterReadyFuncForGraphd(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) {
-	// TODO
-	return true, nil
+func defaultNebulaClusterReadyFuncForGraphd(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
+	isReady := true
+
+	{ // Graphd Resource checks
+		if !isComponentResourceExpected(ctx, cfg, nc.GraphdComponent()) {
+			isReady = false
+		}
+	}
+
+	return isReady, nil
 }
 
-func defaultNebulaClusterReadyFuncForMetad(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) {
-	// TODO
-	return true, nil
+func defaultNebulaClusterReadyFuncForMetad(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
+	isReady := true
+
+	{ // Metad Resource checks
+		if !isComponentResourceExpected(ctx, cfg, nc.MetadComponent()) {
+			isReady = false
+		}
+	}
+
+	return isReady, nil
 }
 
-func defaultNebulaClusterReadyFuncForStoraged(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) {
-	// TODO
-	return true, nil
+func defaultNebulaClusterReadyFuncForStoraged(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
+	isReady := true
+
+	{ // Storaged Resource checks
+		if !isComponentResourceExpected(ctx, cfg, nc.StoragedComponent()) {
+			isReady = false
+		}
+	}
+
+	return isReady, nil
 }
 
 func defaultNebulaClusterReadyFuncForAgent(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) {
@@ -152,7 +176,7 @@ func isComponentStatusExpected(
 ) bool {
 	if errMessages := e2evalidator.StructWithRules(
 		componentStatus,
-		map[string]e2evalidator.Rule{
+		map[string]e2evalidator.Ruler{
 			"Version":                    e2evalidator.Eq(componentSpec.Version),
 			"Phase":                      e2evalidator.Eq(appsv1alpha1.RunningPhase),
 			"Workload.ReadyReplicas":     e2evalidator.Eq(*componentSpec.Replicas),
@@ -173,3 +197,43 @@ func isComponentStatusExpected(
 
 	return true
 }
+
+func isComponentResourceExpected(ctx context.Context, cfg *envconf.Config, component appsv1alpha1.NebulaClusterComponent) bool {
+	sts := &appsv1.StatefulSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      component.GetName(),
+			Namespace: component.GetNamespace(),
+		},
+	}
+
+	if err := cfg.Client().Resources().Get(ctx, sts.Name, sts.Namespace, sts); err != nil {
+		klog.InfoS("Check Component Resource but statefulset not found",
+			"namespace", sts.Namespace,
+			"name", sts.Name,
+		)
+		return false
+	}
+
+	for _, c := range sts.Spec.Template.Spec.Containers {
+		if c.Name != component.ComponentType().String() {
+			continue
+		}
+
+		// TODO: check more fields
+		resource := component.ComponentSpec().Resources()
+		if reflect.DeepEqual(c.Resources.DeepCopy(), resource) {
+			return true
+		} else {
+			klog.InfoS("Check Component Resource but not expected",
+				"namespace", sts.Namespace,
+				"name", sts.Name,
+				"requests", c.Resources.Requests,
+				"requestsExpected", resource.Requests,
+				"limits", c.Resources.Limits,
+				"limitsExpected", resource.Limits,
+			)
+		}
+	}
+
+	return false
+}

tests/e2e/envfuncsext/nebulacluster.go

@@ -87,7 +87,7 @@ func InstallNebulaCluster(opts ...NebulaClusterOption) env.Func {
 		o := (&NebulaClusterOptions{}).
 			WithOptions(
 				WithNebulaClusterHelmOptions(WithHelmOptions(
-				// default values
+					// default values
 				)),
 			).WithOptions(opts...)
 
@@ -110,7 +110,7 @@ func UpgradeNebulaCluster(opts ...NebulaClusterOption) env.Func {
 		o := (&NebulaClusterOptions{}).
 			WithOptions(
 				WithNebulaClusterHelmOptions(WithHelmOptions(
-				// default values
+					// default values
 				)),
 			).WithOptions(opts...)
 
@@ -133,7 +133,7 @@ func WaitNebulaClusterReady(opts ...NebulaClusterOption) env.Func {
 		o := (&NebulaClusterOptions{}).
 			WithOptions(
 				WithNebulaClusterHelmOptions(WithHelmOptions(
-				// default values
+					// default values
 				)),
 			).WithOptions(opts...)
 
@@ -192,7 +192,7 @@ func UninstallNebulaCluster(opts ...NebulaClusterOption) env.Func {
 		o := (&NebulaClusterOptions{}).
 			WithOptions(
 				WithNebulaClusterHelmOptions(WithHelmOptions(
-				// default values
+					// default values
 				)),
 			).WithOptions(opts...)