diff --git a/tests/e2e/e2evalidator/e2evalidator.go b/tests/e2e/e2evalidator/e2evalidator.go index 41806489..6bcf272a 100644 --- a/tests/e2e/e2evalidator/e2evalidator.go +++ b/tests/e2e/e2evalidator/e2evalidator.go @@ -8,9 +8,28 @@ import ( "github.com/go-playground/validator/v10" ) +type Ruler interface { + Cmp(val any) error +} + type Rule string -func StructWithRules(actual any, rulesMapping map[string]Rule) (errMessages []string) { +func (r Rule) Cmp(val any) error { + return validator.New().Var(val, string(r)) +} + +type AnyStruct struct { + Val any +} + +func (a AnyStruct) Cmp(val any) error { + if !reflect.DeepEqual(val, a.Val) { + return fmt.Errorf("objects not equal, expected %v, got %v", a, val) + } + return nil +} + +func StructWithRules(actual any, rulesMapping map[string]Ruler) (errMessages []string) { for field, rule := range rulesMapping { actualVal := reflect.ValueOf(actual) @@ -32,7 +51,8 @@ func StructWithRules(actual any, rulesMapping map[string]Rule) (errMessages []st actualVal = actualVal.Elem() } val := actualVal.Interface() - if err := validator.New().Var(val, string(rule)); err != nil { + + if err := rule.Cmp(val); err != nil { errMessages = append(errMessages, fmt.Sprintf("field %q(%v) does not match %q\n", field, val, rule)) } } diff --git a/tests/e2e/envfuncsext/nebulacluster-ready-func.go b/tests/e2e/envfuncsext/nebulacluster-ready-func.go index d2a36dbb..7411073e 100644 --- a/tests/e2e/envfuncsext/nebulacluster-ready-func.go +++ b/tests/e2e/envfuncsext/nebulacluster-ready-func.go @@ -19,7 +19,10 @@ package envfuncsext import ( "context" stderrors "errors" + "reflect" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "sigs.k8s.io/e2e-framework/pkg/envconf" @@ -46,7 +49,7 @@ func DefaultNebulaClusterReadyFunc(ctx context.Context, cfg *envconf.Config, nc return true, nil } -func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping map[string]e2evalidator.Rule) NebulaClusterReadyFunc { +func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping map[string]e2evalidator.Ruler) NebulaClusterReadyFunc { return func(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (isReady bool, err error) { if errMessages := e2evalidator.StructWithRules(nc, rulesMapping); len(errMessages) > 0 { if ignoreValidationErrors { @@ -61,7 +64,7 @@ func NebulaClusterReadyFuncForFields(ignoreValidationErrors bool, rulesMapping m } } -func defaultNebulaClusterReadyFuncForStatus(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { +func defaultNebulaClusterReadyFuncForStatus(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { isReady := nc.IsReady() if isReady { // TODO: Add more checks @@ -115,19 +118,40 @@ func defaultNebulaClusterReadyFuncForStatus(_ context.Context, _ *envconf.Config return isReady, nil } -func defaultNebulaClusterReadyFuncForGraphd(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) { - // TODO - return true, nil +func defaultNebulaClusterReadyFuncForGraphd(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { + isReady := true + + { // Graphd Resource checks + if !isComponentResourceExpected(ctx, cfg, nc.GraphdComponent()) { + isReady = false + } + } + + return isReady, nil } -func defaultNebulaClusterReadyFuncForMetad(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) { - // TODO - return true, nil +func defaultNebulaClusterReadyFuncForMetad(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { + isReady := true + + { // Metad Resource checks + if !isComponentResourceExpected(ctx, cfg, nc.MetadComponent()) { + isReady = false + } + } + + return isReady, nil } -func defaultNebulaClusterReadyFuncForStoraged(_ context.Context, _ *envconf.Config, _ *appsv1alpha1.NebulaCluster) (bool, error) { - // TODO - return true, nil +func defaultNebulaClusterReadyFuncForStoraged(ctx context.Context, cfg *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { + isReady := true + + { // Storaged Resource checks + if !isComponentResourceExpected(ctx, cfg, nc.StoragedComponent()) { + isReady = false + } + } + + return isReady, nil } func defaultNebulaClusterReadyFuncForAgent(_ context.Context, _ *envconf.Config, nc *appsv1alpha1.NebulaCluster) (bool, error) { @@ -152,7 +176,7 @@ func isComponentStatusExpected( ) bool { if errMessages := e2evalidator.StructWithRules( componentStatus, - map[string]e2evalidator.Rule{ + map[string]e2evalidator.Ruler{ "Version": e2evalidator.Eq(componentSpec.Version), "Phase": e2evalidator.Eq(appsv1alpha1.RunningPhase), "Workload.ReadyReplicas": e2evalidator.Eq(*componentSpec.Replicas), @@ -173,3 +197,43 @@ func isComponentStatusExpected( return true } + +func isComponentResourceExpected(ctx context.Context, cfg *envconf.Config, component appsv1alpha1.NebulaClusterComponent) bool { + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: component.GetName(), + Namespace: component.GetNamespace(), + }, + } + + if err := cfg.Client().Resources().Get(ctx, sts.Name, sts.Namespace, sts); err != nil { + klog.InfoS("Check Component Resource but statefulset not found", + "namespace", sts.Namespace, + "name", sts.Name, + ) + return false + } + + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name != component.ComponentType().String() { + continue + } + + // TODO: check more fields + resource := component.ComponentSpec().Resources() + if reflect.DeepEqual(c.Resources.DeepCopy(), resource) { + return true + } else { + klog.InfoS("Check Component Resource but not expected", + "namespace", sts.Namespace, + "name", sts.Name, + "requests", c.Resources.Requests, + "requestsExpected", resource.Requests, + "limits", c.Resources.Limits, + "limitsExpected", resource.Limits, + ) + } + } + + return false +} diff --git a/tests/e2e/envfuncsext/nebulacluster.go b/tests/e2e/envfuncsext/nebulacluster.go index db07dc67..b6c2b101 100644 --- a/tests/e2e/envfuncsext/nebulacluster.go +++ b/tests/e2e/envfuncsext/nebulacluster.go @@ -87,7 +87,7 @@ func InstallNebulaCluster(opts ...NebulaClusterOption) env.Func { o := (&NebulaClusterOptions{}). WithOptions( WithNebulaClusterHelmOptions(WithHelmOptions( - // default values + // default values )), ).WithOptions(opts...) @@ -110,7 +110,7 @@ func UpgradeNebulaCluster(opts ...NebulaClusterOption) env.Func { o := (&NebulaClusterOptions{}). WithOptions( WithNebulaClusterHelmOptions(WithHelmOptions( - // default values + // default values )), ).WithOptions(opts...) @@ -133,7 +133,7 @@ func WaitNebulaClusterReady(opts ...NebulaClusterOption) env.Func { o := (&NebulaClusterOptions{}). WithOptions( WithNebulaClusterHelmOptions(WithHelmOptions( - // default values + // default values )), ).WithOptions(opts...) @@ -192,7 +192,7 @@ func UninstallNebulaCluster(opts ...NebulaClusterOption) env.Func { o := (&NebulaClusterOptions{}). WithOptions( WithNebulaClusterHelmOptions(WithHelmOptions( - // default values + // default values )), ).WithOptions(opts...) diff --git a/tests/e2e/nebulacluster_basic_test.go b/tests/e2e/nebulacluster_basic_test.go index e807d004..4a0b56a3 100644 --- a/tests/e2e/nebulacluster_basic_test.go +++ b/tests/e2e/nebulacluster_basic_test.go @@ -1,15 +1,19 @@ package e2e import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/e2e-framework/third_party/helm" + "github.com/vesoft-inc/nebula-operator/tests/e2e/e2evalidator" "github.com/vesoft-inc/nebula-operator/tests/e2e/envfuncsext" - "sigs.k8s.io/e2e-framework/third_party/helm" ) const ( - LabelCategoryBasic = "basic" - LabelGroupScale = "scale" - LabelGroupVersion = "version" + LabelCategoryBasic = "basic" + LabelGroupScale = "scale" + LabelGroupVersion = "version" + LabelGroupResources = "resources" ) var testCasesBasic []ncTestCase @@ -32,7 +36,7 @@ var testCasesBasicScale = []ncTestCase{ }, InstallWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(false, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(false, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(2), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -56,7 +60,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(4), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(4), @@ -79,7 +83,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(5), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(4), @@ -102,7 +106,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(5), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(5), @@ -124,7 +128,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(3), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(4), @@ -146,7 +150,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(3), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -168,7 +172,7 @@ var testCasesBasicScale = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(2), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -191,7 +195,7 @@ var testCasesBasicVersion = []ncTestCase{ }, InstallWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(false, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(false, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(2), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -217,7 +221,7 @@ var testCasesBasicVersion = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(2), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -244,7 +248,7 @@ var testCasesBasicVersion = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(4), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(4), @@ -273,7 +277,7 @@ var testCasesBasicVersion = []ncTestCase{ }, UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ envfuncsext.WithNebulaClusterReadyFuncs( - envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Rule{ + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ "Spec.Graphd.Replicas": e2evalidator.Eq(2), "Spec.Metad.Replicas": e2evalidator.Eq(3), "Spec.Storaged.Replicas": e2evalidator.Eq(3), @@ -292,7 +296,156 @@ var testCasesBasicVersion = []ncTestCase{ // test cases about resources of graphd｜metad｜storaged var testCasesBasicResources = []ncTestCase{ - // TODO + { + Name: "update resources", + Labels: map[string]string{ + LabelKeyCategory: LabelCategoryBasic, + LabelKeyGroup: LabelGroupResources, + }, + InstallNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterHelmRawOptions( + helm.WithArgs( + "--set", "nebula.enablePVReclaim=true", + ), + ), + }, + InstallWaitNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterReadyFuncs( + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ + "Spec.Metad.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("1Gi"), + }, + }, + }, + "Spec.Storaged.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("1Gi"), + }, + }, + }, + "Spec.Graphd.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1"), + "memory": resource.MustParse("500Mi"), + }, + }, + }, + }), + envfuncsext.DefaultNebulaClusterReadyFunc, + ), + }, + LoadLDBC: true, + UpgradeCases: []ncTestUpgradeCase{ + { + Name: "update graphd resources", + UpgradeNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterHelmRawOptions( + helm.WithArgs( + "--set", "nebula.graphd.resources.limits.cpu=1100m", + "--set", "nebula.graphd.resources.limits.memory=1100Mi", + ), + ), + }, + UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterReadyFuncs( + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ + "Spec.Graphd.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1100m"), + "memory": resource.MustParse("1100Mi"), + }, + }, + }, + }), + envfuncsext.DefaultNebulaClusterReadyFunc, + ), + }, + }, + { + Name: "update metad resources", + UpgradeNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterHelmRawOptions( + helm.WithArgs( + "--set", "nebula.metad.resources.limits.cpu=1100m", + "--set", "nebula.metad.resources.limits.memory=1100Mi", + ), + ), + }, + UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterReadyFuncs( + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ + "Spec.Metad.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1100m"), + "memory": resource.MustParse("1100Mi"), + }, + }, + }, + }), + envfuncsext.DefaultNebulaClusterReadyFunc, + ), + }, + }, + { + Name: "update storaged resources", + UpgradeNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterHelmRawOptions( + helm.WithArgs( + "--set", "nebula.storaged.resources.limits.cpu=1100m", + "--set", "nebula.storaged.resources.limits.memory=1100Mi", + ), + ), + }, + UpgradeWaitNCOptions: []envfuncsext.NebulaClusterOption{ + envfuncsext.WithNebulaClusterReadyFuncs( + envfuncsext.NebulaClusterReadyFuncForFields(true, map[string]e2evalidator.Ruler{ + "Spec.Storaged.Resources": &e2evalidator.AnyStruct{ + Val: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "cpu": resource.MustParse("500m"), + "memory": resource.MustParse("500Mi"), + }, + Limits: corev1.ResourceList{ + "cpu": resource.MustParse("1100m"), + "memory": resource.MustParse("1100Mi"), + }, + }, + }, + }), + envfuncsext.DefaultNebulaClusterReadyFunc, + ), + }, + }, + }, + }, } // test cases about image of graphd｜metad｜storaged