From 7ac1d749af9b116876504cc2aec0793a7f0c23d1 Mon Sep 17 00:00:00 2001 From: Morven Cao Date: Thu, 29 Aug 2024 17:05:29 +0800 Subject: [PATCH] fix e2e testing. (#184) Signed-off-by: morvencao --- Makefile | 3 +- go.mod | 4 +- go.sum | 11 +- test/e2e/pkg/consumer_test.go | 86 ++++---- test/e2e/pkg/grpc_test.go | 80 +++----- test/e2e/pkg/resources_test.go | 74 +++---- test/e2e/pkg/serverside_test.go | 9 +- test/e2e/pkg/sourceclient_test.go | 60 +++--- test/e2e/pkg/spec_resync_test.go | 236 +++++++++++----------- test/e2e/pkg/status_resync_test.go | 58 +++--- test/e2e/pkg/suite_test.go | 46 +++++ test/factories.go | 271 ++++++++++++-------------- test/integration/consumers_test.go | 6 +- test/integration/controller_test.go | 3 +- test/integration/pulse_server_test.go | 4 +- test/integration/resource_test.go | 35 ++-- 16 files changed, 513 insertions(+), 473 deletions(-) diff --git a/Makefile b/Makefile index 5126a8d0..4b9a1705 100755 --- a/Makefile +++ b/Makefile @@ -417,7 +417,8 @@ e2e-test/teardown: .PHONY: e2e-test/teardown e2e-test: e2e-test/teardown e2e-test/setup - ginkgo -v --output-dir="${PWD}/test/e2e/report" --json-report=report.json --junit-report=report.xml \ + ginkgo -v --fail-fast --label-filter="!(e2e-tests-spec-resync-reconnect||e2e-tests-status-resync-reconnect)" \ + --output-dir="${PWD}/test/e2e/report" --json-report=report.json --junit-report=report.xml \ ${PWD}/test/e2e/pkg -- \ -api-server=https://$(shell cat ${PWD}/test/e2e/.external_host_ip):30080 \ -grpc-server=$(shell cat ${PWD}/test/e2e/.external_host_ip):30090 \ diff --git a/go.mod b/go.mod index 3d6bd08b..c5fc151d 100755 --- a/go.mod +++ b/go.mod @@ -46,10 +46,12 @@ require ( k8s.io/klog/v2 v2.120.1 open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c open-cluster-management.io/ocm v0.13.1-0.20240618054845-e2a7b9e78b33 - open-cluster-management.io/sdk-go v0.14.1-0.20240806021439-bf354ff3847f + open-cluster-management.io/sdk-go v0.14.1-0.20240829071054-7bd852f2b2a8 ) require ( + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4 v0.0.0-20200712162734-eb1adaa8a7a6 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect diff --git a/go.sum b/go.sum index 23a78743..1ab30f89 100755 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -539,8 +538,8 @@ go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= @@ -825,10 +824,8 @@ open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c h1:gYfgkX/U open-cluster-management.io/api v0.14.1-0.20240627145512-bd6f2229b53c/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM= open-cluster-management.io/ocm v0.13.1-0.20240618054845-e2a7b9e78b33 h1:7uPjyn1x25QZIzfZqeSFfZdNrzc2hlHm6t/JKYKu9fI= open-cluster-management.io/ocm v0.13.1-0.20240618054845-e2a7b9e78b33/go.mod h1:KzUwhPZAg6Wq+4xRu10fVVpqNADyz5CtRW4ziqIC2z4= -open-cluster-management.io/sdk-go v0.14.1-0.20240717021054-955108a181ee h1:aQ4AoR8SKz/byOyZbbYC9Tbp4VCtRHje8uHbn438o84= -open-cluster-management.io/sdk-go v0.14.1-0.20240717021054-955108a181ee/go.mod h1:xFmN3Db5nN68oLGnstmIRv4us8HJCdXFnBNMXVp0jWY= -open-cluster-management.io/sdk-go v0.14.1-0.20240806021439-bf354ff3847f h1:8yQ8uFemyM/dI4nMo8pVOmEiIn156SkN9qpnLf8UOcI= -open-cluster-management.io/sdk-go v0.14.1-0.20240806021439-bf354ff3847f/go.mod h1:xFmN3Db5nN68oLGnstmIRv4us8HJCdXFnBNMXVp0jWY= +open-cluster-management.io/sdk-go v0.14.1-0.20240829071054-7bd852f2b2a8 h1:2dOKe8kj2niAZMlc75NSI/CIkosNNt/Kqyau7ZH4DwM= +open-cluster-management.io/sdk-go v0.14.1-0.20240829071054-7bd852f2b2a8/go.mod h1:mHGre2DnTfV5gLgCWr+byBqKqTuf5Yzx/EtSSJ2EiGE= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= diff --git a/test/e2e/pkg/consumer_test.go b/test/e2e/pkg/consumer_test.go index 96ed84fb..c0c5b732 100644 --- a/test/e2e/pkg/consumer_test.go +++ b/test/e2e/pkg/consumer_test.go @@ -1,44 +1,67 @@ package e2e_test import ( + "fmt" "net/http" "reflect" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/openshift-online/maestro/pkg/api/openapi" + "k8s.io/apimachinery/pkg/util/rand" ) -var _ = Describe("Consumer", Ordered, func() { - var consumer openapi.Consumer - var resourceConsumer openapi.Consumer - var resource openapi.Resource - BeforeAll(func() { - consumer = openapi.Consumer{Name: openapi.PtrString("linda")} - resourceConsumer = openapi.Consumer{Name: openapi.PtrString("susan")} - resource = helper.NewAPIResource(*resourceConsumer.Name, 1) - }) - +var _ = Describe("Consumers", Ordered, Label("e2e-tests-consumers"), func() { Context("Consumer CRUD Tests", func() { + consumerA := openapi.Consumer{Name: openapi.PtrString("consumer-a")} + consumerB := openapi.Consumer{Name: openapi.PtrString("consumer-b")} + resource := helper.NewAPIResource(*consumerB.Name, fmt.Sprintf("nginx-%s", rand.String(5)), 1) + + AfterAll(func() { + // delete the consumer + resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *consumerA.Id).Execute() + Expect(err).NotTo(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + + _, resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumerA.Id).Execute() + Expect(err.Error()).To(ContainSubstring("Not Found")) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + + // delete the consumer associated with resource + resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *consumerB.Id).Execute() + Expect(err).To(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) // 403 forbid deletion + + // delete the resource on the consumer + resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource.Id).Execute() + Expect(err).To(Succeed()) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + + // only if permanently delete the resource, the consumer can be deleted + resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *consumerB.Id).Execute() + Expect(err).To(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) // 403 forbid deletion + }) + It("create consumer", func() { // create a consumer without resource - created, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersPost(ctx).Consumer(consumer).Execute() + created, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersPost(ctx).Consumer(consumerA).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) Expect(*created.Id).NotTo(BeEmpty()) - consumer = *created + consumerA = *created - got, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumer.Id).Execute() + got, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumerA.Id).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) Expect(got).NotTo(BeNil()) // create a consumer with resource - created, resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersPost(ctx).Consumer(resourceConsumer).Execute() + created, resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersPost(ctx).Consumer(consumerB).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) Expect(*created.Id).NotTo(BeEmpty()) - resourceConsumer = *created + consumerB = *created res, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(resource).Execute() Expect(err).ShouldNot(HaveOccurred()) @@ -54,10 +77,11 @@ var _ = Describe("Consumer", Ordered, func() { Expect(resp.StatusCode).To(Equal(http.StatusOK)) Expect(consumerList).NotTo(BeNil()) Expect(len(consumerList.Items) > 0).To(BeTrue()) + fmt.Printf("consumer list: %v\n", consumerList.Items) got := false for _, c := range consumerList.Items { - if *c.Name == *consumer.Name { + if *c.Name == *consumerA.Name { got = true } } @@ -66,45 +90,19 @@ var _ = Describe("Consumer", Ordered, func() { It("patch consumer", func() { labels := &map[string]string{"hello": "world"} - patched, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdPatch(ctx, *consumer.Id). + patched, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdPatch(ctx, *consumerA.Id). ConsumerPatchRequest(openapi.ConsumerPatchRequest{Labels: labels}).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) _, ok := patched.GetLabelsOk() Expect(ok).To(BeTrue()) - got, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumer.Id).Execute() + got, resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumerA.Id).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) Expect(got).NotTo(BeNil()) eq := reflect.DeepEqual(*labels, *got.Labels) Expect(eq).To(BeTrue()) }) - - AfterAll(func() { - // delete the consumer - resp, err := apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *consumer.Id).Execute() - Expect(err).NotTo(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) - - _, resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdGet(ctx, *consumer.Id).Execute() - Expect(err.Error()).To(ContainSubstring("Not Found")) - Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) - - // delete the consumer associated with resource - resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *resourceConsumer.Id).Execute() - Expect(err).To(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) // 403 forbid deletion - - // delete the resource on the consumer - resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource.Id).Execute() - Expect(err).To(Succeed()) - Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) - - // only if permanently delete the resource, the consumer can be deleted - resp, err = apiClient.DefaultApi.ApiMaestroV1ConsumersIdDelete(ctx, *resourceConsumer.Id).Execute() - Expect(err).To(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) // 403 forbid deletion - }) }) }) diff --git a/test/e2e/pkg/grpc_test.go b/test/e2e/pkg/grpc_test.go index e2342f3e..8d5c6386 100644 --- a/test/e2e/pkg/grpc_test.go +++ b/test/e2e/pkg/grpc_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "time" @@ -17,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" pbv1 "open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc/protobuf/v1" grpcprotocol "open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc/protocol" @@ -28,6 +28,7 @@ import ( var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { Context("GRPC Manifest Tests", func() { source := "grpc-e2e" + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) resourceID := uuid.NewString() resourceStatus := &api.ResourceStatus{ ReconcileStatus: &api.ReconcileStatus{}, @@ -95,14 +96,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource spec using grpc client", func() { - evt, err := helper.ManifestToEvent(1, source, "create_request", consumer.Name, resourceID, 1, false) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewEvent(source, "create_request", consumer.Name, resourceID, deployName, 1, 1) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -136,7 +133,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -156,14 +153,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource spec with update request using grpc client", func() { - evt, err := helper.ManifestToEvent(2, source, "update_request", consumer.Name, resourceID, 1, false) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewEvent(source, "update_request", consumer.Name, resourceID, deployName, 1, 2) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -197,7 +190,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -217,14 +210,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource spec with delete request using grpc client", func() { - evt, err := helper.ManifestToEvent(2, source, "delete_request", consumer.Name, resourceID, 1, true) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewEvent(source, "delete_request", consumer.Name, resourceID, deployName, 2, 2) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -245,7 +234,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -258,13 +247,14 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the resource with the maestro api", func() { _, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdGet(ctx, resourceID).Execute() - Expect(err).To(HaveOccurred(), "Expected 404") + Expect(err).To(HaveOccurred(), "Expected 404 error") Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) }) }) Context("GRPC Manifest Bundle Tests", func() { source := "grpc-e2e" + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) resourceID := uuid.NewString() resourceBundleStatus := &api.ResourceBundleStatus{ ManifestBundleStatus: &payload.ManifestBundleStatus{}, @@ -314,14 +304,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource bundle spec using grpc client", func() { - evt, err := helper.ManifestsToBundleEvent(1, source, "create_request", consumer.Name, resourceID, 1, false) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewBundleEvent(source, "create_request", consumer.Name, resourceID, deployName, 1, 1) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -370,7 +356,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -390,14 +376,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource bundle spec with update request using grpc client", func() { - evt, err := helper.ManifestsToBundleEvent(2, source, "update_request", consumer.Name, resourceID, 1, false) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewBundleEvent(source, "update_request", consumer.Name, resourceID, deployName, 1, 2) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -446,7 +428,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -466,14 +448,10 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { }) It("publish a resource bundle spec with delete request using grpc client", func() { - evt, err := helper.ManifestsToBundleEvent(2, source, "delete_request", consumer.Name, resourceID, 1, true) - Expect(err).ShouldNot(HaveOccurred()) - + evt := helper.NewBundleEvent(source, "delete_request", consumer.Name, resourceID, deployName, 2, 2) pbEvt := &pbv1.CloudEvent{} - if err = grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt); err != nil { - log.Fatalf("failed to convert spec from cloudevent to protobuf: %v", err) - } - + err := grpcprotocol.WritePBMessage(ctx, binding.ToMessage(evt), pbEvt) + Expect(err).To(BeNil(), "failed to convert spec from cloudevent to protobuf") _, err = grpcClient.Publish(ctx, &pbv1.PublishRequest{Event: pbEvt}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -494,7 +472,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the nginx deployment from cluster", func() { Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -507,7 +485,7 @@ var _ = Describe("GRPC", Ordered, Label("e2e-tests-grpc"), func() { It("get the resource with the maestro api", func() { _, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourceBundlesIdGet(ctx, resourceID).Execute() - Expect(err).To(HaveOccurred(), "Expected 404") + Expect(err).To(HaveOccurred(), "Expected 404 error") Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) }) }) diff --git a/test/e2e/pkg/resources_test.go b/test/e2e/pkg/resources_test.go index 876e8a95..2ca7aef8 100644 --- a/test/e2e/pkg/resources_test.go +++ b/test/e2e/pkg/resources_test.go @@ -20,13 +20,12 @@ import ( "github.com/openshift-online/maestro/pkg/api/openapi" ) -// go test -v ./test/e2e/pkg -args -api-server=$api_server -consumer-name=$consumer.Name -consumer-kubeconfig=$consumer_kubeconfig -ginkgo.focus "Resources" var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { - var resource *openapi.Resource - Context("Resource CRUD Tests", func() { + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + var resource *openapi.Resource It("post the nginx resource to the maestro api", func() { - res := helper.NewAPIResource(consumer.Name, 1) + res := helper.NewAPIResource(consumer.Name, deployName, 1) var resp *http.Response var err error resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() @@ -36,7 +35,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Expect(*resource.Version).To(Equal(int32(1))) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -56,7 +55,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) It("patch the nginx resource with the maestro api", func() { - newRes := helper.NewAPIResource(consumer.Name, 2) + newRes := helper.NewAPIResource(consumer.Name, deployName, 2) patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resource.Id). ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resource.Version, Manifest: newRes.Manifest}).Execute() Expect(err).ShouldNot(HaveOccurred()) @@ -64,7 +63,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Expect(*patchedResource.Version).To(Equal(*resource.Version + 1)) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -81,7 +80,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -94,18 +93,20 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) Context("Resource Delete Option Tests", func() { - res := helper.NewAPIResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + var resource *openapi.Resource It("post the nginx resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployName, 1) + res.DeleteOption = map[string]interface{}{"propagationPolicy": "Orphan"} var resp *http.Response var err error - res.DeleteOption = map[string]interface{}{"propagationPolicy": "Orphan"} resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) Expect(*resource.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -123,7 +124,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { // ensure the "nginx" deployment in the "default" namespace is not deleted Consistently(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return fmt.Errorf("nginx deployment is deleted") @@ -134,11 +135,11 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) It("delete the nginx deployment", func() { - err := consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err := consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, deployName, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -151,18 +152,20 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) Context("Resource CreateOnly UpdateStrategy Tests", func() { + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + var resource *openapi.Resource It("post the nginx resource to the maestro api with createOnly updateStrategy", func() { - res := helper.NewAPIResource(consumer.Name, 1) + res := helper.NewAPIResource(consumer.Name, deployName, 1) + res.UpdateStrategy = map[string]interface{}{"type": "CreateOnly"} var resp *http.Response var err error - res.UpdateStrategy = map[string]interface{}{"type": "CreateOnly"} resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) Expect(*resource.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return err } @@ -174,15 +177,16 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) It("patch the nginx resource", func() { - newRes := helper.NewAPIResource(consumer.Name, 2) + newRes := helper.NewAPIResource(consumer.Name, deployName, 2) patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resource.Id). ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resource.Version, Manifest: newRes.Manifest}).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) Expect(*patchedResource.Version).To(Equal(*resource.Version + 1)) + // ensure the "nginx" deployment in the "default" namespace is not updated Consistently(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { return nil } @@ -199,7 +203,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -212,16 +216,18 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { }) Context("Resource ReadOnly UpdateStrategy Tests via restful api", func() { - It("create a sample deployment in the target cluster", func() { + var resource *openapi.Resource + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + It("create a nginx deployment in the target cluster", func() { nginxDeploy := &appsv1.Deployment{} - err := json.Unmarshal(helper.GetTestNginxJSON(1), nginxDeploy) + err := json.Unmarshal([]byte(helper.NewResourceManifestJSON(deployName, 1)), nginxDeploy) Expect(err).ShouldNot(HaveOccurred()) _, err = consumer.ClientSet.AppsV1().Deployments("default").Create(ctx, nginxDeploy, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) }) It("post the resource to the maestro api with readonly updateStrategy", func() { - res := helper.NewReadOnlyAPIResource(consumer.Name) + res := helper.NewReadOnlyAPIResource(consumer.Name, deployName) var resp *http.Response var err error resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() @@ -254,6 +260,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { return nil } } + return fmt.Errorf("contentStatus should not be empty") }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) @@ -263,11 +270,11 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) - err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, deployName, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -281,8 +288,9 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Context("Resource ReadOnly UpdateStrategy Tests via gRPC", func() { workName := "work-readonly-" + rand.String(5) - secretName := "auth" - It("create a sample secret in the target cluster", func() { + secretName := "auth-" + rand.String(5) + manifest := fmt.Sprintf("{\"apiVersion\":\"v1\",\"kind\":\"Secret\",\"metadata\":{\"name\":\"%s\",\"namespace\":\"default\"}}", secretName) + It("create a secret in the target cluster", func() { _, err := consumer.ClientSet.CoreV1().Secrets("default").Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, @@ -305,7 +313,7 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { Manifests: []workv1.Manifest{ { RawExtension: runtime.RawExtension{ - Raw: []byte("{\"apiVersion\":\"v1\",\"kind\":\"Secret\",\"metadata\":{\"name\":\"auth\",\"namespace\":\"default\"}}"), + Raw: []byte(manifest), }, }, }, @@ -364,16 +372,16 @@ var _ = Describe("Resources", Ordered, Label("e2e-tests-resources"), func() { return fmt.Errorf("work creationTimestamp is empty") } - manifest := work.Status.ResourceStatus.Manifests - if len(manifest) > 0 && len(manifest[0].StatusFeedbacks.Values) != 0 { - feedback := manifest[0].StatusFeedbacks.Values + manifests := work.Status.ResourceStatus.Manifests + if len(manifests) > 0 && len(manifests[0].StatusFeedbacks.Values) != 0 { + feedback := manifests[0].StatusFeedbacks.Values if feedback[0].Name == "credential" && *feedback[0].Value.JsonRaw == "{\"token\":\"dG9rZW4=\"}" { return nil } - return fmt.Errorf("the result %v is not expected", feedback[0]) + return fmt.Errorf("the status feedback value %v is not expected", feedback[0]) } - return fmt.Errorf("manifest should be empty") + return fmt.Errorf("manifests are empty") }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) diff --git a/test/e2e/pkg/serverside_test.go b/test/e2e/pkg/serverside_test.go index e5d13a9f..ed07f6c6 100644 --- a/test/e2e/pkg/serverside_test.go +++ b/test/e2e/pkg/serverside_test.go @@ -13,6 +13,7 @@ import ( "github.com/openshift-online/maestro/pkg/api/openapi" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/util/rand" workv1 "open-cluster-management.io/api/work/v1" ) @@ -22,7 +23,7 @@ const sleepJob = ` "apiVersion": "batch/v1", "kind": "Job", "metadata": { - "name": "sleep", + "name": "%s", "namespace": "default" }, "spec": { @@ -47,14 +48,16 @@ const sleepJob = ` } ` -var _ = Describe("Server Side Apply", func() { +var _ = Describe("Server Side Apply", Ordered, Label("e2e-tests-serverside-apply"), func() { It("Apply a job with maestro", func() { // The kube-apiserver will set a default selector and label on the Pod of Job if the job does not have // spec.Selector, these fields are immutable, if we use update strategy to apply Job, it will report // AppliedManifestFailed. The maestro uses the server side strategy to apply a resource with ManifestWork // by default, this will avoid this. manifest := map[string]interface{}{} - Expect(json.Unmarshal([]byte(sleepJob), &manifest)).ShouldNot(HaveOccurred()) + sleepJobName := fmt.Sprintf("sleep-%s", rand.String(5)) + err := json.Unmarshal([]byte(fmt.Sprintf(sleepJob, sleepJobName)), &manifest) + Expect(err).ShouldNot(HaveOccurred()) res := openapi.Resource{ Manifest: manifest, diff --git a/test/e2e/pkg/sourceclient_test.go b/test/e2e/pkg/sourceclient_test.go index 6993a0b0..ec6a0d20 100644 --- a/test/e2e/pkg/sourceclient_test.go +++ b/test/e2e/pkg/sourceclient_test.go @@ -25,7 +25,7 @@ import ( "open-cluster-management.io/sdk-go/pkg/cloudevents/work/common" ) -var _ = Describe("gRPC Source ManifestWork Client Test", func() { +var _ = Describe("Source ManifestWork Client", Ordered, Label("e2e-tests-source-work-client"), func() { Context("Update an obsolete work", func() { var workName string @@ -35,7 +35,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { Eventually(func() error { _, err := workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) return err - }, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) // wait for few seconds to ensure the creation is finished <-time.After(5 * time.Second) @@ -47,7 +47,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { Eventually(func() error { return AssertWorkNotFound(workName) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) @@ -73,6 +73,9 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { _, err = workClient.ManifestWorks(consumer.Name).Patch(ctx, workName, types.MergePatchType, patchData, metav1.PatchOptions{}) Expect(err).Should(HaveOccurred()) Expect(strings.Contains(err.Error(), "the resource version is not the latest")).Should(BeTrue()) + + // wait for few seconds to ensure the update is finished + <-time.After(5 * time.Second) }) }) @@ -93,7 +96,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { _, err := workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - initWorkBName = "init-work-b" + rand.String(5) + initWorkBName = "init-work-b-" + rand.String(5) work = NewManifestWorkWithLabels(initWorkBName, map[string]string{"app": "test"}) _, err = workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) @@ -112,7 +115,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { } return AssertWorkNotFound(initWorkBName) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) watcherCancel() }) @@ -161,7 +164,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { Eventually(func() error { return AssertWatchResult(result) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) It("The watchers with different namespace", func() { @@ -202,7 +205,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { Eventually(func() error { return AssertWatchResult(allConsumerWatcherResult) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) Eventually(func() error { return AssertWatchResult(consumerWatcherResult) @@ -247,7 +250,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { Eventually(func() error { return AssertWatchResult(result) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) }) @@ -265,25 +268,28 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { _, err := workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - prodWorkName = "work-prod" + rand.String(5) - work = NewManifestWorkWithLabels(prodWorkName, map[string]string{"app": "nginx", "env": "prod"}) + prodWorkName = "work-production" + rand.String(5) + work = NewManifestWorkWithLabels(prodWorkName, map[string]string{"app": "test", "env": "production"}) _, err = workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - testWorkAName = "work-test-a-" + rand.String(5) - work = NewManifestWorkWithLabels(testWorkAName, map[string]string{"app": "nginx", "env": "test", "val": "a"}) + testWorkAName = "work-integration-a-" + rand.String(5) + work = NewManifestWorkWithLabels(testWorkAName, map[string]string{"app": "test", "env": "integration", "val": "a"}) _, err = workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - testWorkBName = "work-test-b-" + rand.String(5) - work = NewManifestWorkWithLabels(testWorkBName, map[string]string{"app": "nginx", "env": "test", "val": "b"}) + testWorkBName = "work-integration-b-" + rand.String(5) + work = NewManifestWorkWithLabels(testWorkBName, map[string]string{"app": "test", "env": "integration", "val": "b"}) _, err = workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) - testWorkCName = "work-test-c-" + rand.String(5) - work = NewManifestWorkWithLabels(testWorkCName, map[string]string{"app": "nginx", "env": "test", "val": "c"}) + testWorkCName = "work-integration-c-" + rand.String(5) + work = NewManifestWorkWithLabels(testWorkCName, map[string]string{"app": "test", "env": "integration", "val": "c"}) _, err = workClient.ManifestWorks(consumer.Name).Create(ctx, work, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) + + // wait for few seconds to ensure the creation is finished + <-time.After(5 * time.Second) }) AfterEach(func() { @@ -320,7 +326,7 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { } return AssertWorkNotFound(testWorkCName) - }, 30*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) }) It("List works with options", func() { @@ -348,28 +354,28 @@ var _ = Describe("gRPC Source ManifestWork Client Test", func() { By("list works with app label") works, err = workClient.ManifestWorks(consumer.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "app=nginx", + LabelSelector: "app=test", }) Expect(err).ShouldNot(HaveOccurred()) Expect(AssertWorks(works.Items, prodWorkName, testWorkAName, testWorkBName, testWorkCName)).ShouldNot(HaveOccurred()) By("list works without test env") works, err = workClient.ManifestWorks(consumer.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "app=nginx,env!=test", + LabelSelector: "app=test,env!=integration", }) Expect(err).ShouldNot(HaveOccurred()) Expect(AssertWorks(works.Items, prodWorkName)).ShouldNot(HaveOccurred()) By("list works in prod and test env") works, err = workClient.ManifestWorks(consumer.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "env in (prod, test)", + LabelSelector: "env in (production, integration)", }) Expect(err).ShouldNot(HaveOccurred()) Expect(AssertWorks(works.Items, prodWorkName, testWorkAName, testWorkBName, testWorkCName)).ShouldNot(HaveOccurred()) By("list works in test env and val not in a and b") works, err = workClient.ManifestWorks(consumer.Name).List(ctx, metav1.ListOptions{ - LabelSelector: "env=test,val notin (a,b)", + LabelSelector: "env=integration,val notin (a,b)", }) Expect(err).ShouldNot(HaveOccurred()) Expect(AssertWorks(works.Items, testWorkCName)).ShouldNot(HaveOccurred()) @@ -501,6 +507,12 @@ func AssertWorks(works []workv1.ManifestWork, expected ...string) error { return nil } +func NewManifestWorkWithLabels(name string, labels map[string]string) *workv1.ManifestWork { + work := NewManifestWork(name) + work.Labels = labels + return work +} + func NewManifestWork(name string) *workv1.ManifestWork { return &workv1.ManifestWork{ ObjectMeta: metav1.ObjectMeta{ @@ -516,12 +528,6 @@ func NewManifestWork(name string) *workv1.ManifestWork { } } -func NewManifestWorkWithLabels(name string, labels map[string]string) *workv1.ManifestWork { - work := NewManifestWork(name) - work.Labels = labels - return work -} - func NewManifest(name string) workv1.Manifest { obj := &unstructured.Unstructured{ Object: map[string]interface{}{ diff --git a/test/e2e/pkg/spec_resync_test.go b/test/e2e/pkg/spec_resync_test.go index 5891b20c..5d374a43 100644 --- a/test/e2e/pkg/spec_resync_test.go +++ b/test/e2e/pkg/spec_resync_test.go @@ -13,50 +13,53 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" ) -var _ = Describe("Spec resync", Ordered, Label("e2e-tests-spec-resync"), func() { - var resource1, resource2, resource3 *openapi.Resource - var mqttReplicas, maestroServerReplicas, maestroAgentReplicas int - +var _ = Describe("Spec Resync After Restart", Ordered, Label("e2e-tests-spec-resync-restart"), func() { Context("Resource resync resource spec after maestro agent restarts", func() { - It("post the nginx-1 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 1) + var maestroAgentReplicas int + var resourceA, resourceB, resourceC *openapi.Resource + deployA := fmt.Sprintf("nginx-%s", rand.String(5)) + deployB := fmt.Sprintf("nginx-%s", rand.String(5)) + deployC := fmt.Sprintf("nginx-%s", rand.String(5)) + It("post the nginx A resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployA, 1) var resp *http.Response var err error - resource1, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceA, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource1.Id).ShouldNot(BeEmpty()) + Expect(*resourceA.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 1, got %d", deployA, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("post the nginx-2 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 2) + It("post the nginx B resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployB, 1) var resp *http.Response var err error - resource2, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceB, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource2.Id).ShouldNot(BeEmpty()) + Expect(*resourceB.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-2 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx B deployment %s, expected 1, got %d", deployB, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) @@ -89,71 +92,68 @@ var _ = Describe("Spec resync", Ordered, Label("e2e-tests-spec-resync"), func() }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("patch the nginx-1 resource", func() { - newRes := helper.NewAPIResourceWithIndex(consumer.Name, 2, 1) - patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resource1.Id). - ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resource1.Version, Manifest: newRes.Manifest}).Execute() + It("patch the nginx A resource", func() { + newRes := helper.NewAPIResource(consumer.Name, deployA, 2) + patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resourceA.Id). + ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resourceA.Version, Manifest: newRes.Manifest}).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusOK)) - Expect(*patchedResource.Version).To(Equal(*resource1.Version + 1)) + Expect(*patchedResource.Version).To(Equal(*resourceA.Version + 1)) }) - It("ensure the nginx-1 resource is not updated", func() { - // ensure the "nginx-1" deployment in the "default" namespace is not updated + It("ensure the nginx A resource is not updated", func() { Consistently(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return nil } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 1, got %d", deployA, *deploy.Spec.Replicas) } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("delete the nginx-2 resource", func() { - resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource2.Id).Execute() + It("delete the nginx B resource", func() { + resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceB.Id).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) }) - It("ensure the nginx-2 resource is not deleted", func() { - // ensure the "nginx-2" deployment in the "default" namespace is not deleted + It("ensure the nginx B resource is not deleted", func() { Consistently(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - return fmt.Errorf("nginx-2 deployment is deleted") + return fmt.Errorf("nginx B deployment %s is deleted", deployB) } } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("post the nginx-3 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 3) + It("post the nginx C resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployC, 1) var resp *http.Response var err error - resource3, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceC, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource3.Id).ShouldNot(BeEmpty()) + Expect(*resourceC.Id).ShouldNot(BeEmpty()) }) - It("ensure the nginx-3 resource is not created", func() { - // ensure the "nginx-3" deployment in the "default" namespace is not created + It("ensure the nginx C resource is not created", func() { Consistently(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err == nil { - return fmt.Errorf("nginx-3 deployment is created") + return fmt.Errorf("nginx C deployment %s is created", deployC) } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("start maestro agent", func() { - // patch maestro agent replicas to maestroAgentReplicas + It("restart maestro agent", func() { + // patch maestro agent replicas back deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro-agent").Patch(ctx, "maestro-agent", types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, maestroAgentReplicas)), metav1.PatchOptions{ FieldManager: "testconsumer.ClientSet", }) @@ -183,116 +183,123 @@ var _ = Describe("Spec resync", Ordered, Label("e2e-tests-spec-resync"), func() }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-1 resource is updated", func() { + It("ensure the nginx A resource is updated", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 2 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 2, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 2, got %d", deployA, *deploy.Spec.Replicas) } return nil - }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }, 3*time.Minute, 3*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-2 resource is deleted", func() { + It("ensure the nginx B resource is deleted", func() { Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-2 deployment still exists") + return fmt.Errorf("nginx B deployment %s still exists", deployB) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-3 resource is created", func() { + It("ensure the nginx C resource is created", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx C deployment %s, expected 1, got %d", deployC, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("delete the nginx-1 and nginx-3 resource", func() { - resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource1.Id).Execute() - Expect(err).ShouldNot(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) - - resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource3.Id).Execute() + It("delete the nginx A and nginx C resources", func() { + resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceA.Id).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-1 deployment still exists") + return fmt.Errorf("nginx A deployment %s still exists", deployA) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceC.Id).Execute() + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-3 deployment still exists") + return fmt.Errorf("nginx C deployment %s still exists", deployC) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) }) +}) +var _ = Describe("Spec Resync After Reconnect", Ordered, Label("e2e-tests-spec-resync-reconnect"), func() { Context("Resource resync resource spec after maestro agent reconnects", func() { - It("post the nginx-1 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 1) + var maestroServerReplicas, mqttReplicas int + var resourceA, resourceB, resourceC *openapi.Resource + deployA := fmt.Sprintf("nginx-%s", rand.String(5)) + deployB := fmt.Sprintf("nginx-%s", rand.String(5)) + deployC := fmt.Sprintf("nginx-%s", rand.String(5)) + It("post the nginx A resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployA, 1) var resp *http.Response var err error - resource1, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceA, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource1.Id).ShouldNot(BeEmpty()) + Expect(*resourceA.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 1, got %d", deployA, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("post the nginx-2 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 2) + It("post the nginx B resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployB, 1) var resp *http.Response var err error - resource2, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceB, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource2.Id).ShouldNot(BeEmpty()) + Expect(*resourceB.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-2 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx B deployment %s, expected 1, got %d", deployB, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) @@ -416,73 +423,70 @@ var _ = Describe("Spec resync", Ordered, Label("e2e-tests-spec-resync"), func() }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("patch the nginx-1 resource", func() { - newRes := helper.NewAPIResourceWithIndex(consumer.Name, 2, 1) + It("patch the nginx A resource", func() { + newRes := helper.NewAPIResource(consumer.Name, deployA, 2) Eventually(func() error { - patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resource1.Id). - ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resource1.Version, Manifest: newRes.Manifest}).Execute() + patchedResource, resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, *resourceA.Id). + ResourcePatchRequest(openapi.ResourcePatchRequest{Version: resourceA.Version, Manifest: newRes.Manifest}).Execute() if err != nil { return err } if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code, expected 200, got %d", resp.StatusCode) } - if *patchedResource.Version != *resource1.Version+1 { - return fmt.Errorf("unexpected version, expected %d, got %d", *resource1.Version+1, *patchedResource.Version) + if *patchedResource.Version != *resourceA.Version+1 { + return fmt.Errorf("unexpected version, expected %d, got %d", *resourceA.Version+1, *patchedResource.Version) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-1 resource is not updated", func() { - // ensure the "nginx-1" deployment in the "default" namespace is not updated + It("ensure the nginx A resource is not updated", func() { Consistently(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return nil } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 1, got %d", deployA, *deploy.Spec.Replicas) } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("delete the nginx-2 resource", func() { - resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource2.Id).Execute() + It("delete the nginx B resource", func() { + resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceB.Id).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) }) - It("ensure the nginx-2 resource is not deleted", func() { - // ensure the "nginx-2" deployment in the "default" namespace is not deleted + It("ensure the nginx B resource is not deleted", func() { Consistently(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - return fmt.Errorf("nginx-2 deployment is deleted") + return fmt.Errorf("nginx B deployment %s is deleted", deployB) } } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("post the nginx-3 resource to the maestro api", func() { - res := helper.NewAPIResourceWithIndex(consumer.Name, 1, 3) + It("post the nginx C resource to the maestro api", func() { + res := helper.NewAPIResource(consumer.Name, deployC, 1) var resp *http.Response var err error - resource3, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() + resourceC, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) - Expect(*resource3.Id).ShouldNot(BeEmpty()) + Expect(*resourceC.Id).ShouldNot(BeEmpty()) }) - It("ensure the nginx-3 resource is not created", func() { - // ensure the "nginx-3" deployment in the "default" namespace is not created + It("ensure the nginx C resource is not created", func() { Consistently(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err == nil { - return fmt.Errorf("nginx-3 deployment is created") + return fmt.Errorf("nginx C deployment %s is created", deployC) } return nil }, 10*time.Second, 1*time.Second).ShouldNot(HaveOccurred()) @@ -539,74 +543,74 @@ var _ = Describe("Spec resync", Ordered, Label("e2e-tests-spec-resync"), func() Expect(err).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-1 resource is updated", func() { + It("ensure the nginx A resource is updated", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 2 { - return fmt.Errorf("unexpected replicas for nginx-1 deployment, expected 2, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx A deployment %s, expected 2, got %d", deployA, *deploy.Spec.Replicas) } return nil }, 3*time.Minute, 3*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-2 resource is deleted", func() { + It("ensure the nginx B resource is deleted", func() { Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-2", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployB, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-2 deployment still exists") + return fmt.Errorf("nginx B deployment %s still exists", deployB) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("ensure the nginx-3 resource is created", func() { + It("ensure the nginx C resource is created", func() { Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx C deployment %s, expected 1, got %d", deployC, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("delete the nginx-1 and nginx-3 resource", func() { - resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource1.Id).Execute() - Expect(err).ShouldNot(HaveOccurred()) - Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) - - resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resource3.Id).Execute() + It("delete the nginx A and nginx C resources", func() { + resp, err := apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceA.Id).Execute() Expect(err).ShouldNot(HaveOccurred()) Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-1", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployA, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-1 deployment still exists") + return fmt.Errorf("nginx A deployment %s still exists", deployA) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesIdDelete(ctx, *resourceC.Id).Execute() + Expect(err).ShouldNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) + Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx-3", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, deployC, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil } return err } - return fmt.Errorf("nginx-3 deployment still exists") + return fmt.Errorf("nginx C deployment %s still exists", deployC) }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) }) diff --git a/test/e2e/pkg/status_resync_test.go b/test/e2e/pkg/status_resync_test.go index baa88525..a2292333 100644 --- a/test/e2e/pkg/status_resync_test.go +++ b/test/e2e/pkg/status_resync_test.go @@ -15,15 +15,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/rand" ) -var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), func() { - var resource *openapi.Resource - var mqttReplicas, maestroServerReplicas int - +var _ = Describe("Status Resync After Restart", Ordered, Label("e2e-tests-status-resync-restart"), func() { Context("Resource resync resource status after maestro server restarts", func() { + var maestroServerReplicas int + var resource *openapi.Resource + name := fmt.Sprintf("nginx-%s", rand.String(5)) It("post the nginx resource with non-default service account to the maestro api", func() { - res := helper.NewAPIResourceWithSA(consumer.Name, 1, "nginx") + res := helper.NewAPIResourceWithSA(consumer.Name, name, name, 1) var resp *http.Response var err error resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() @@ -32,12 +33,12 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun Expect(*resource.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx deployment %s, expected 1, got %d", name, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) @@ -61,7 +62,7 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun return fmt.Errorf("unexpected status, expected error looking up service account default/nginx, got %s", string(statusJSON)) } return nil - }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) }) It("shut down maestro server", func() { @@ -91,21 +92,21 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) }) - It("create default/nginx serviceaccount", func() { + It("create serviceaccount for nginx deployment", func() { _, err := consumer.ClientSet.CoreV1().ServiceAccounts("default").Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", + Name: name, }, }, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) // delete the nginx deployment to tigger recreating - err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, name, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) }) - It("start maestro server", func() { - // patch maestro server replicas to 1 + It("restart maestro server", func() { + // patch maestro server replicas back deploy, err := consumer.ClientSet.AppsV1().Deployments("maestro").Patch(ctx, "maestro", types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, maestroServerReplicas)), metav1.PatchOptions{ FieldManager: "testConsumer.ClientSet", }) @@ -161,7 +162,7 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -169,16 +170,21 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun return err } return fmt.Errorf("nginx deployment still exists") - }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) - err = consumer.ClientSet.CoreV1().ServiceAccounts("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err = consumer.ClientSet.CoreV1().ServiceAccounts("default").Delete(ctx, name, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) }) }) +}) +var _ = Describe("Status Resync After Reconnect", Ordered, Label("e2e-tests-status-resync-reconnect"), func() { Context("Resource resync resource status after maestro server reconnects", func() { + var mqttReplicas int + var resource *openapi.Resource + name := fmt.Sprintf("nginx-%s", rand.String(5)) It("post the nginx resource with non-default service account to the maestro api", func() { - res := helper.NewAPIResourceWithSA(consumer.Name, 1, "nginx") + res := helper.NewAPIResourceWithSA(consumer.Name, name, name, 1) var resp *http.Response var err error resource, resp, err = apiClient.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() @@ -187,12 +193,12 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun Expect(*resource.Id).ShouldNot(BeEmpty()) Eventually(func() error { - deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + deploy, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } if *deploy.Spec.Replicas != 1 { - return fmt.Errorf("unexpected replicas, expected 1, got %d", *deploy.Spec.Replicas) + return fmt.Errorf("unexpected replicas for nginx deployment %s, expected 1, got %d", name, *deploy.Spec.Replicas) } return nil }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) @@ -216,7 +222,7 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun return fmt.Errorf("unexpected status, expected error looking up service account default/nginx, got %s", string(statusJSON)) } return nil - }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) }) It("delete the mqtt-broker service for server", func() { @@ -224,16 +230,16 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun Expect(err).ShouldNot(HaveOccurred()) }) - It("create default/nginx serviceaccount", func() { + It("create serviceaccount for nginx deployment", func() { _, err := consumer.ClientSet.CoreV1().ServiceAccounts("default").Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: "nginx", + Name: name, }, }, metav1.CreateOptions{}) Expect(err).ShouldNot(HaveOccurred()) // delete the nginx deployment to tigger recreating - err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err = consumer.ClientSet.AppsV1().Deployments("default").Delete(ctx, name, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) }) @@ -343,7 +349,7 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) Eventually(func() error { - _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, "nginx", metav1.GetOptions{}) + _, err := consumer.ClientSet.AppsV1().Deployments("default").Get(ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return nil @@ -351,9 +357,9 @@ var _ = Describe("Status resync", Ordered, Label("e2e-tests-status-resync"), fun return err } return fmt.Errorf("nginx deployment still exists") - }, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred()) + }, 2*time.Minute, 2*time.Second).ShouldNot(HaveOccurred()) - err = consumer.ClientSet.CoreV1().ServiceAccounts("default").Delete(ctx, "nginx", metav1.DeleteOptions{}) + err = consumer.ClientSet.CoreV1().ServiceAccounts("default").Delete(ctx, name, metav1.DeleteOptions{}) Expect(err).ShouldNot(HaveOccurred()) }) }) diff --git a/test/e2e/pkg/suite_test.go b/test/e2e/pkg/suite_test.go index dc185781..39cf809c 100644 --- a/test/e2e/pkg/suite_test.go +++ b/test/e2e/pkg/suite_test.go @@ -1,9 +1,11 @@ package e2e_test import ( + "bytes" "context" "crypto/tls" "flag" + "fmt" "log" "net/http" "testing" @@ -11,6 +13,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + matav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -110,6 +114,8 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { + // dump debug info + dumpDebugInfo() grpcConn.Close() cancel() }) @@ -119,3 +125,43 @@ type ConsumerOptions struct { KubeConfig string ClientSet *kubernetes.Clientset } + +func dumpDebugInfo() { + // dump the maestro server logs + dumpPodLogs(ctx, consumer.ClientSet, "app=maestro", "maestro") + // dump the maestro agent ogs + dumpPodLogs(ctx, consumer.ClientSet, "app=maestro-agent", "maestro-agent") +} + +func dumpPodLogs(ctx context.Context, kubeClient kubernetes.Interface, podSelector, podNamespace string) error { + // get pods from podSelector + pods, err := kubeClient.CoreV1().Pods(podNamespace).List(ctx, matav1.ListOptions{LabelSelector: podSelector}) + if err != nil { + return fmt.Errorf("failed to list pods with pod selector (%s): %v", podSelector, err) + } + + for _, pod := range pods.Items { + logReq := kubeClient.CoreV1().Pods(podNamespace).GetLogs(pod.Name, &corev1.PodLogOptions{}) + logs, err := logReq.Stream(context.Background()) + if err != nil { + return fmt.Errorf("failed to open log stream: %v", err) + } + defer logs.Close() + + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(logs) + if err != nil { + return fmt.Errorf("failed to read pod logs: %v", err) + } + + log.Printf("=========================================== POD LOGS START ===========================================") + log.Printf("Pod %s/%s phase: %s", pod.Name, podNamespace, string(pod.Status.Phase)) + for _, containerStatus := range pod.Status.ContainerStatuses { + log.Printf("Container %s status: %v", containerStatus.Name, containerStatus.State) + } + log.Printf("Pod %s/%s logs: \n%s", pod.Name, podNamespace, buf.String()) + log.Printf("=========================================== POD LOGS STOP ===========================================") + } + + return nil +} diff --git a/test/factories.go b/test/factories.go index d9331a20..bbfd07c3 100755 --- a/test/factories.go +++ b/test/factories.go @@ -13,6 +13,7 @@ import ( "gorm.io/datatypes" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" workv1 "open-cluster-management.io/api/work/v1" cetypes "open-cluster-management.io/sdk-go/pkg/cloudevents/generic/types" @@ -24,42 +25,8 @@ var testManifestJSON = ` "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": "nginx", - "namespace": "default" - }, - "spec": { - "replicas": %d, - "selector": { - "matchLabels": { - "app": "nginx" - } - }, - "template": { - "metadata": { - "labels": { - "app": "nginx" - } - }, - "spec": { - "containers": [ - { - "image": "nginxinc/nginx-unprivileged", - "name": "nginx" - } - ] - } - } - } -} -` - -var testManifestJSONWithSA = ` -{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "nginx", - "namespace": "default" + "name": "%s", + "namespace": "%s" }, "spec": { "replicas": %d, @@ -88,47 +55,13 @@ var testManifestJSONWithSA = ` } ` -var testManifestIndexJSON = ` -{ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": "nginx-%d", - "namespace": "default" - }, - "spec": { - "replicas": %d, - "selector": { - "matchLabels": { - "app": "nginx" - } - }, - "template": { - "metadata": { - "labels": { - "app": "nginx" - } - }, - "spec": { - "containers": [ - { - "image": "nginxinc/nginx-unprivileged", - "name": "nginx" - } - ] - } - } - } -} -` - var testReadOnlyManifestJSON = ` { "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": "nginx", - "namespace": "default" + "name": "%s", + "namespace": "%s" }, "update_strategy": { "type": "ReadOnly" @@ -136,34 +69,21 @@ var testReadOnlyManifestJSON = ` } ` -func (helper *Helper) NewAPIResource(consumerName string, replicas int) openapi.Resource { - testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, replicas)), &testManifest); err != nil { - helper.T.Errorf("error unmarshalling test manifest: %q", err) - } - - return openapi.Resource{ - Manifest: testManifest, - ConsumerName: &consumerName, - } -} - -func (helper *Helper) NewAPIResourceWithSA(consumerName string, replicas int, sa string) openapi.Resource { - testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSONWithSA, replicas, sa)), &testManifest); err != nil { - helper.T.Errorf("error unmarshalling test manifest: %q", err) - } - - return openapi.Resource{ - Manifest: testManifest, - ConsumerName: &consumerName, - } +// NewAPIResource creates an API resource with the given consumer name, deploy name, and replicas. +// It generates a deployment for nginx using the testManifestJSON template, giving it a random deploy +// name to avoid testing conflicts. +func (helper *Helper) NewAPIResource(consumerName, deployName string, replicas int) openapi.Resource { + sa := "default" // default service account + return helper.NewAPIResourceWithSA(consumerName, deployName, sa, replicas) } -func (helper *Helper) NewAPIResourceWithIndex(consumerName string, replicas, index int) openapi.Resource { +// NewAPIResourceWithSA creates an API resource with the given consumer name, deploy name, service account, and replicas. +// It generates a nginx deployment using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +func (helper *Helper) NewAPIResourceWithSA(consumerName, deployName, sa string, replicas int) openapi.Resource { + namespace := "default" // default namespace testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestIndexJSON, index, replicas)), &testManifest); err != nil { - helper.T.Errorf("error unmarshalling test manifest: %q", err) + if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, deployName, namespace, replicas, sa)), &testManifest); err != nil { + helper.T.Errorf("error unmarshalling manifest: %q", err) } return openapi.Resource{ @@ -172,13 +92,22 @@ func (helper *Helper) NewAPIResourceWithIndex(consumerName string, replicas, ind } } -func (helper *Helper) GetTestNginxJSON(replicas int) []byte { - return []byte(fmt.Sprintf(testManifestJSON, replicas)) +// NewResourceManifestJSON creates a resource manifest in JSON format with the given deploy name and replicas. +// It generates a deployment for nginx using the testManifestJSON template, assigning a random deploy name to avoid +// testing conflicts. +func (helper *Helper) NewResourceManifestJSON(deployName string, replicas int) string { + namespace := "default" // default namespace + sa := "default" // default service account + return fmt.Sprintf(testManifestJSON, deployName, namespace, replicas, sa) } -func (helper *Helper) NewReadOnlyAPIResource(consumerName string) openapi.Resource { +// NewReadOnlyAPIResource creates an API resource with the given consumer name and deploy name. +// It generates a read-only deployment manifests for nginx using the testReadOnlyManifestJSON template, +// giving it a random deploy name to avoid testing conflicts. +func (helper *Helper) NewReadOnlyAPIResource(consumerName, deployName string) openapi.Resource { + namespace := "default" // default namespace testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprint(testReadOnlyManifestJSON)), &testManifest); err != nil { + if err := json.Unmarshal([]byte(fmt.Sprintf(testReadOnlyManifestJSON, deployName, namespace)), &testManifest); err != nil { helper.T.Errorf("error unmarshalling test manifest: %q", err) } @@ -188,8 +117,10 @@ func (helper *Helper) NewReadOnlyAPIResource(consumerName string) openapi.Resour } } -func (helper *Helper) NewResource(consumerName string, replicas int) *api.Resource { - testResource := helper.NewAPIResource(consumerName, replicas) +// NewReadOnlyResourceManifestJSON creates a resource with the given consumer name, deploy name, replicas, and resource version. +// It generates a deployment for nginx using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +func (helper *Helper) NewResource(consumerName, deployName string, replicas int, resourceVersion int32) *api.Resource { + testResource := helper.NewAPIResource(consumerName, deployName, replicas) testPayload, err := api.EncodeManifest(testResource.Manifest, testResource.DeleteOption, testResource.UpdateStrategy) if err != nil { helper.T.Errorf("error encoding manifest: %q", err) @@ -199,15 +130,17 @@ func (helper *Helper) NewResource(consumerName string, replicas int) *api.Resour ConsumerName: consumerName, Type: api.ResourceTypeSingle, Payload: testPayload, - Version: 1, + Version: resourceVersion, } return resource } -func (helper *Helper) CreateResource(consumerName string, replicas int) *api.Resource { +// CreateResource creates a resource with the given consumer name, deploy name, and replicas. +// It generates a deployment for nginx using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +func (helper *Helper) CreateResource(consumerName, deployName string, replicas int) *api.Resource { + resource := helper.NewResource(consumerName, deployName, replicas, 1) resourceService := helper.Env().Services.Resources() - resource := helper.NewResource(consumerName, replicas) res, err := resourceService.Create(context.Background(), resource) if err != nil { @@ -217,30 +150,43 @@ func (helper *Helper) CreateResource(consumerName string, replicas int) *api.Res return res } +// CreateResourceList generates a list of resources with the specified consumer name and count. +// Each resource gets a randomly generated deploy name for nginx deployments to avoid testing conflicts. func (helper *Helper) CreateResourceList(consumerName string, count int) (resources []*api.Resource) { for i := 1; i <= count; i++ { - resources = append(resources, helper.CreateResource(consumerName, 1)) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + resources = append(resources, helper.CreateResource(consumerName, deployName, 1)) time.Sleep(10 * time.Millisecond) } + return resources } -// EncodeManifestBundle converts resource manifests into a CloudEvent JSONMap representation. -func (helper *Helper) EncodeManifestBundle(manifest map[string]interface{}) (datatypes.JSONMap, error) { - if len(manifest) == 0 { +// EncodeManifestBundle converts resource manifest JSON into a CloudEvent JSONMap representation. +func (helper *Helper) EncodeManifestBundle(manifestJSON, deployName, deployNamespace string) (datatypes.JSONMap, error) { + if len(manifestJSON) == 0 { return nil, nil } + // unmarshal manifest JSON + manifest := map[string]interface{}{} + if err := json.Unmarshal([]byte(manifestJSON), &manifest); err != nil { + return nil, fmt.Errorf("error unmarshalling manifest: %v", err) + } + + // default deletion option delOption := &workv1.DeleteOption{ PropagationPolicy: workv1.DeletePropagationPolicyTypeForeground, } + // default update strategy upStrategy := &workv1.UpdateStrategy{ Type: workv1.UpdateStrategyTypeServerSideApply, } + source := "maestro" // create a cloud event with the manifest as the data - evt := cetypes.NewEventBuilder("maestro", cetypes.CloudEventsType{}).NewEvent() + evt := cetypes.NewEventBuilder(source, cetypes.CloudEventsType{}).NewEvent() eventPayload := &workpayload.ManifestBundle{ Manifests: []workv1.Manifest{ { @@ -267,47 +213,51 @@ func (helper *Helper) EncodeManifestBundle(manifest map[string]interface{}) (dat ResourceIdentifier: workv1.ResourceIdentifier{ Group: "apps", Resource: "deployments", - Name: "nginx", - Namespace: "default", + Name: deployName, + Namespace: deployNamespace, }, }, }, } + // set the event data if err := evt.SetData(cloudevents.ApplicationJSON, eventPayload); err != nil { return nil, fmt.Errorf("failed to set cloud event data: %v", err) } // convert cloudevent to JSONMap - manifest, err := api.CloudEventToJSONMap(&evt) + manifestBundle, err := api.CloudEventToJSONMap(&evt) if err != nil { return nil, fmt.Errorf("failed to convert cloudevent to resource manifest: %v", err) } - return manifest, nil + return manifestBundle, nil } -func (helper *Helper) NewResourceBundle(name, consumerName string, replicas int) *api.Resource { - testResource := helper.NewAPIResource(consumerName, replicas) - testPayload, err := helper.EncodeManifestBundle(testResource.Manifest) +// NewResourceBundle creates a resource bundle with the given consumer name, deploy name, replicas, and resource version. +func (helper *Helper) NewResourceBundle(consumerName, deployName string, replicas int, resourceVersion int32) *api.Resource { + namespace := "default" // default namespace + manifestJSON := helper.NewResourceManifestJSON(deployName, replicas) + payload, err := helper.EncodeManifestBundle(manifestJSON, deployName, namespace) if err != nil { helper.T.Errorf("error encoding manifest bundle: %q", err) } resource := &api.Resource{ - Name: name, ConsumerName: consumerName, Type: api.ResourceTypeBundle, - Payload: testPayload, - Version: 1, + Payload: payload, + Version: resourceVersion, } return resource } -func (helper *Helper) CreateResourceBundle(name, consumerName string, replicas int) *api.Resource { +// CreateResourceBundle creates a resource bundle with the given consumer name, deploy name and replicas. +// It generates a deployment for nginx using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +func (helper *Helper) CreateResourceBundle(consumerName, deployName string, replicas int) *api.Resource { + resourceBundle := helper.NewResourceBundle(consumerName, deployName, replicas, 1) resourceService := helper.Env().Services.Resources() - resourceBundle := helper.NewResourceBundle(name, consumerName, replicas) res, err := resourceService.Create(context.Background(), resourceBundle) if err != nil { @@ -317,11 +267,15 @@ func (helper *Helper) CreateResourceBundle(name, consumerName string, replicas i return res } -func (helper *Helper) CreateResourceBundleList(consumerName string, count int) (resources []*api.Resource) { +// CreateResourceBundleList generates a list of resource bundles with the specified consumer name and count. +// Each resource gets a randomly generated deploy name for nginx deployments to avoid testing conflicts. +func (helper *Helper) CreateResourceBundleList(consumerName string, count int) (resourceBundles []*api.Resource) { for i := 1; i <= count; i++ { - resources = append(resources, helper.CreateResourceBundle(fmt.Sprintf("resource%d", i), consumerName, 1)) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + resourceBundles = append(resourceBundles, helper.CreateResourceBundle(consumerName, deployName, 1)) } - return resources + + return resourceBundles } func (helper *Helper) CreateConsumer(name string) *api.Consumer { @@ -333,7 +287,7 @@ func (helper *Helper) CreateConsumerWithLabels(name string, labels map[string]st consumer, err := consumerService.Create(context.Background(), &api.Consumer{Name: name, Labels: db.EmptyMapToNilStringMap(&labels)}) if err != nil { - helper.T.Errorf("error creating resource: %q", err) + helper.T.Errorf("error creating consumer: %q", err) } return consumer } @@ -342,16 +296,19 @@ func (helper *Helper) CreateConsumerList(count int) (consumers []*api.Consumer) for i := 1; i <= count; i++ { consumers = append(consumers, helper.CreateConsumer(fmt.Sprintf("consumer-%d", i))) } + return consumers } -// ManifestToEvent converts a manifest into a CloudEvent representation with manifest data. -func (helper *Helper) ManifestToEvent(replicas int, source, action, consumerName, resourceID string, - resourceVersion int64, deleting bool) (*cloudevents.Event, error) { - +// NewEvent creates a CloudEvent with the given source, action, consumer name, resource ID, deploy name, resource version, and replicas. +// It generates a nginx deployment using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +// If the action is "delete_request," the event includes a deletion timestamp. +func (helper *Helper) NewEvent(source, action, consumerName, resourceID, deployName string, resourceVersion int64, replicas int) *cloudevents.Event { + sa := "default" // default service account + deployNamespace := "default" // default namespace testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, replicas)), &testManifest); err != nil { - return nil, fmt.Errorf("error unmarshalling test manifest: %v", err) + if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, deployName, deployNamespace, replicas, sa)), &testManifest); err != nil { + helper.T.Errorf("error unmarshalling manifest: %q", err) } eventType := cetypes.CloudEventsType{ @@ -363,11 +320,20 @@ func (helper *Helper) ManifestToEvent(replicas int, source, action, consumerName WithClusterName(consumerName). WithResourceID(resourceID). WithResourceVersion(resourceVersion) - if deleting { + + // add deletion timestamp if action is delete_request + if action == "delete_request" { evtBuilder.WithDeletionTimestamp(time.Now()) } + evt := evtBuilder.NewEvent() + // if action is delete_request, no data is needed + if action == "delete_request" { + evt.SetData(cloudevents.ApplicationJSON, nil) + return &evt + } + eventPayload := &workpayload.Manifest{ Manifest: unstructured.Unstructured{Object: testManifest}, DeleteOption: &workv1.DeleteOption{ @@ -392,19 +358,21 @@ func (helper *Helper) ManifestToEvent(replicas int, source, action, consumerName } if err := evt.SetData(cloudevents.ApplicationJSON, eventPayload); err != nil { - return nil, fmt.Errorf("failed to set cloud event data: %v", err) + helper.T.Errorf("failed to set cloud event data: %q", err) } - return &evt, nil + return &evt } -// ManifestsToBundleEvent converts a list of manifests into a CloudEvent representation with manifest bundle data. -func (helper *Helper) ManifestsToBundleEvent(replicas int, source, action, consumerName, resourceID string, - resourceVersion int64, deleting bool) (*cloudevents.Event, error) { - +// NewBundleEvent creates a CloudEvent with the given source, action, consumer name, resource ID, resource version, and replicas. +// It generates a bundle of nginx deployments using the testManifestJSON template, assigning a random deploy name to avoid testing conflicts. +// If the action is "delete_request," the event includes a deletion timestamp. +func (helper *Helper) NewBundleEvent(source, action, consumerName, resourceID, deployName string, resourceVersion int64, replicas int) *cloudevents.Event { + sa := "default" // default service account + deployNamespace := "default" // default namespace testManifest := map[string]interface{}{} - if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, replicas)), &testManifest); err != nil { - return nil, fmt.Errorf("error unmarshalling test manifest: %v", err) + if err := json.Unmarshal([]byte(fmt.Sprintf(testManifestJSON, deployName, deployNamespace, replicas, sa)), &testManifest); err != nil { + helper.T.Errorf("error unmarshalling manifest: %q", err) } eventType := cetypes.CloudEventsType{ @@ -418,11 +386,20 @@ func (helper *Helper) ManifestsToBundleEvent(replicas int, source, action, consu WithClusterName(consumerName). WithResourceID(resourceID). WithResourceVersion(resourceVersion) - if deleting { + + // add deletion timestamp if action is delete_request + if action == "delete_request" { evtBuilder.WithDeletionTimestamp(time.Now()) } + evt := evtBuilder.NewEvent() + // if action is delete_request, no data is needed + if action == "delete_request" { + evt.SetData(cloudevents.ApplicationJSON, nil) + return &evt + } + eventPayload := &workpayload.ManifestBundle{ Manifests: []workv1.Manifest{ { @@ -453,16 +430,16 @@ func (helper *Helper) ManifestsToBundleEvent(replicas int, source, action, consu ResourceIdentifier: workv1.ResourceIdentifier{ Group: "apps", Resource: "deployments", - Name: "nginx", - Namespace: "default", + Name: deployName, + Namespace: deployNamespace, }, }, }, } if err := evt.SetData(cloudevents.ApplicationJSON, eventPayload); err != nil { - return nil, fmt.Errorf("failed to set cloud event data: %v", err) + helper.T.Errorf("failed to set cloud event data: %q", err) } - return &evt, nil + return &evt } diff --git a/test/integration/consumers_test.go b/test/integration/consumers_test.go index bbb75bbc..284da9a1 100644 --- a/test/integration/consumers_test.go +++ b/test/integration/consumers_test.go @@ -188,7 +188,8 @@ func TestConsumerDeleteForbidden(t *testing.T) { Expect(*consumer.Id).NotTo(BeEmpty()) // attach resource to the consumer - res := h.NewAPIResource(*consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewAPIResource(*consumer.Name, deployName, 1) resource, resp, err := client.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() Expect(err).To(Succeed()) Expect(resp.StatusCode).To(Equal(http.StatusCreated)) @@ -245,7 +246,8 @@ func TestConsumerDeleting(t *testing.T) { go func(name, id string) { defer wg.Done() for i := 0; i < resourceNum; i++ { - res := h.NewAPIResource(name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewAPIResource(name, deployName, 1) resource, resp, err := client.DefaultApi.ApiMaestroV1ResourcesPost(ctx).Resource(res).Execute() resourceChan <- &Result{ resource: resource, diff --git a/test/integration/controller_test.go b/test/integration/controller_test.go index 74c6c51b..a6ce56cc 100755 --- a/test/integration/controller_test.go +++ b/test/integration/controller_test.go @@ -202,7 +202,8 @@ func TestControllerReconcile(t *testing.T) { time.Sleep(100 * time.Millisecond) consumer := h.CreateConsumer("cluster-" + rand.String(5)) - resource := h.CreateResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + resource := h.CreateResource(consumer.Name, deployName, 1) // Eventually, the event will be processed by the controller. Eventually(func() error { diff --git a/test/integration/pulse_server_test.go b/test/integration/pulse_server_test.go index d838b47a..9556f2bd 100644 --- a/test/integration/pulse_server_test.go +++ b/test/integration/pulse_server_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" workv1 "open-cluster-management.io/api/work/v1" "github.com/openshift-online/maestro/pkg/api" @@ -71,7 +72,8 @@ func TestPulseServer(t *testing.T) { }) Expect(err).NotTo(HaveOccurred()) - res := h.CreateResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.CreateResource(consumer.Name, deployName, 1) h.StartControllerManager(ctx) h.StartWorkAgent(ctx, consumer.Name, false) clientHolder := h.WorkAgentHolder diff --git a/test/integration/resource_test.go b/test/integration/resource_test.go index 50165557..5e9d86ce 100755 --- a/test/integration/resource_test.go +++ b/test/integration/resource_test.go @@ -44,7 +44,8 @@ func TestResourceGet(t *testing.T) { Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) consumer := h.CreateConsumer("cluster-" + rand.String(5)) - resource := h.CreateResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + resource := h.CreateResource(consumer.Name, deployName, 1) res, resp, err := client.DefaultApi.ApiMaestroV1ResourcesIdGet(ctx, resource.ID).Execute() Expect(err).NotTo(HaveOccurred()) @@ -68,7 +69,8 @@ func TestResourcePost(t *testing.T) { clusterName := "cluster-" + rand.String(5) consumer := h.CreateConsumer(clusterName) - res := h.NewAPIResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewAPIResource(consumer.Name, deployName, 1) h.StartControllerManager(ctx) h.StartWorkAgent(ctx, consumer.Name, false) clientHolder := h.WorkAgentHolder @@ -189,7 +191,8 @@ func TestResourcePostWithoutName(t *testing.T) { clusterName := "cluster-" + rand.String(5) consumer := h.CreateConsumer(clusterName) - res := h.NewAPIResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewAPIResource(consumer.Name, deployName, 1) h.StartControllerManager(ctx) resourceService := h.Env().Services.Resources() // POST responses per openapi spec: 201, 400, 409, 500 @@ -230,7 +233,8 @@ func TestResourcePostWithName(t *testing.T) { clusterName := "cluster-" + rand.String(5) consumer := h.CreateConsumer(clusterName) - res := h.NewAPIResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewAPIResource(consumer.Name, deployName, 1) h.StartControllerManager(ctx) // POST responses per openapi spec: 201, 400, 409, 500 @@ -266,7 +270,8 @@ func TestResourcePatch(t *testing.T) { clientHolder := h.WorkAgentHolder agentWorkClient := clientHolder.ManifestWorks(consumer.ID) - res := h.CreateResource(consumer.ID, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.CreateResource(consumer.ID, deployName, 1) Expect(res.Version).To(Equal(int32(1))) var work *workv1.ManifestWork @@ -294,7 +299,7 @@ func TestResourcePatch(t *testing.T) { }, 20*time.Second, 2*time.Second).Should(Succeed()) // 200 OK - newRes := h.NewAPIResource(consumer.ID, 2) + newRes := h.NewAPIResource(consumer.Name, deployName, 2) resource, resp, err := client.DefaultApi.ApiMaestroV1ResourcesIdPatch(ctx, res.ID).ResourcePatchRequest(openapi.ResourcePatchRequest{Version: &res.Version, Manifest: newRes.Manifest}).Execute() Expect(err).NotTo(HaveOccurred(), "Error posting object: %v", err) Expect(resp.StatusCode).To(Equal(http.StatusOK)) @@ -433,7 +438,8 @@ func TestResourceBundleGet(t *testing.T) { Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) consumer := h.CreateConsumer("cluster-" + rand.String(5)) - resourceBundle := h.CreateResourceBundle("resource1", consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + resourceBundle := h.CreateResourceBundle(consumer.Name, deployName, 1) resBundle, resp, err := client.DefaultApi.ApiMaestroV1ResourceBundlesIdGet(ctx, resourceBundle.ID).Execute() Expect(err).NotTo(HaveOccurred()) @@ -482,8 +488,9 @@ func TestUpdateResourceWithRacingRequests(t *testing.T) { ctx := h.NewAuthenticatedContext(account) consumer := h.CreateConsumer("cluster-" + rand.String(5)) - res := h.CreateResource(consumer.Name, 1) - newRes := h.NewAPIResource(consumer.Name, 2) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.CreateResource(consumer.Name, deployName, 1) + newRes := h.NewAPIResource(consumer.Name, deployName, 2) // starts 20 threads to update this resource at the same time threads := 20 @@ -549,7 +556,8 @@ func TestResourceFromGRPC(t *testing.T) { // create a mock resource clusterName := "cluster-" + rand.String(5) consumer := h.CreateConsumer(clusterName) - res := h.NewResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewResource(consumer.Name, deployName, 1, 1) res.ID = uuid.NewString() h.StartControllerManager(ctx) @@ -667,7 +675,7 @@ func TestResourceFromGRPC(t *testing.T) { return nil }, 10*time.Second, 1*time.Second).Should(Succeed()) - newRes := h.NewResource(consumer.Name, 2) + newRes := h.NewResource(consumer.Name, deployName, 2, 1) newRes.ID = *resource.Id newRes.Version = *resource.Version err = h.GRPCSourceClient.Publish(ctx, types.CloudEventsType{ @@ -759,7 +767,8 @@ func TestResourceBundleFromGRPC(t *testing.T) { // create a mock resource clusterName := "cluster-" + rand.String(5) consumer := h.CreateConsumer(clusterName) - res := h.NewResource(consumer.Name, 1) + deployName := fmt.Sprintf("nginx-%s", rand.String(5)) + res := h.NewResource(consumer.Name, deployName, 1, 1) res.ID = uuid.NewString() h.StartControllerManager(ctx) @@ -864,7 +873,7 @@ func TestResourceBundleFromGRPC(t *testing.T) { return nil }, 10*time.Second, 1*time.Second).Should(Succeed()) - newRes := h.NewResource(consumer.Name, 2) + newRes := h.NewResource(consumer.Name, deployName, 2, 1) newRes.ID = res.ID err = h.GRPCSourceClient.Publish(ctx, types.CloudEventsType{ CloudEventsDataType: payload.ManifestBundleEventDataType,