From f80044d73f34ec65a369d12cf10593beb6c477a9 Mon Sep 17 00:00:00 2001 From: Daniel Hrabovcak Date: Fri, 29 Sep 2023 11:02:49 -0400 Subject: [PATCH] Automatically add subtest labels for cleanup --- e2e/kubeclient.go | 194 +++++++++++++++++++++++++++++++++++ e2e/operator_context_test.go | 58 +++-------- 2 files changed, 211 insertions(+), 41 deletions(-) create mode 100644 e2e/kubeclient.go diff --git a/e2e/kubeclient.go b/e2e/kubeclient.go new file mode 100644 index 0000000000..2db8bcbcd3 --- /dev/null +++ b/e2e/kubeclient.go @@ -0,0 +1,194 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package e2e contains tests that validate the behavior of gmp-operator against a cluster. +// To make tests simple and fast, the test suite runs the operator internally. The CRDs +// are expected to be installed out of band (along with the operator deployment itself in +// a real world setup). +package e2e + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type DelegatingClient interface { + client.Client + Base() client.Client +} + +// delegatingWriteSubResourceClient delegates all mutating functions to a writer client. +type delegatingWriteSubResourceClient struct { + base client.SubResourceClient + writer client.SubResourceWriter +} + +func (c *delegatingWriteSubResourceClient) Get(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error { + return c.base.Get(ctx, obj, subResource, opts...) +} + +func (c *delegatingWriteSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + return c.writer.Create(ctx, obj, subResource, opts...) +} + +func (c *delegatingWriteSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return c.writer.Update(ctx, obj, opts...) +} + +func (c *delegatingWriteSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + return c.writer.Patch(ctx, obj, patch, opts...) +} + +// WriterClient comprises of all mutating functions in client.Client. +type WriterClient interface { + client.Writer + client.StatusClient + SubResource(subResource string) client.SubResourceWriter +} + +// delegatingWriteClient delegates all mutating functions to a writer client. +type delegatingWriteClient struct { + base client.Client + writer WriterClient +} + +func (c *delegatingWriteClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return c.base.Get(ctx, key, obj, opts...) +} + +func (c *delegatingWriteClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return c.base.List(ctx, list, opts...) +} + +func (c *delegatingWriteClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return c.writer.Create(ctx, obj, opts...) +} + +func (c *delegatingWriteClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return c.writer.Delete(ctx, obj, opts...) +} + +func (c *delegatingWriteClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return c.writer.Update(ctx, obj, opts...) +} + +func (c *delegatingWriteClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return c.writer.Patch(ctx, obj, patch, opts...) +} + +func (c *delegatingWriteClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return c.writer.DeleteAllOf(ctx, obj, opts...) +} + +func (c *delegatingWriteClient) SubResource(subResource string) client.SubResourceClient { + return &delegatingWriteSubResourceClient{ + base: c.base.SubResource(subResource), + writer: c.writer.SubResource(subResource), + } +} + +func (c *delegatingWriteClient) Status() client.SubResourceWriter { + return c.writer.Status() +} + +func (c *delegatingWriteClient) Scheme() *runtime.Scheme { + return c.base.Scheme() +} + +func (c *delegatingWriteClient) RESTMapper() meta.RESTMapper { + return c.base.RESTMapper() +} + +func (c *delegatingWriteClient) Base() client.Client { + return c.base +} + +func newDelegatingWriteClient(c client.Client, w WriterClient) DelegatingClient { + return &delegatingWriteClient{ + base: c, + writer: w, + } +} + +// labelWriterClient adds common labels to all objects mutated by this client. +type labelWriterClient struct { + base client.Client + labels map[string]string +} + +func (c *labelWriterClient) setLabels(obj client.Object) { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + for k, v := range c.labels { + labels[k] = v + } + obj.SetLabels(labels) +} + +func (c *labelWriterClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + c.setLabels(obj) + return c.base.Create(ctx, obj, opts...) +} + +func (c *labelWriterClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return c.base.Delete(ctx, obj, opts...) +} + +func (c *labelWriterClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + c.setLabels(obj) + return c.base.Update(ctx, obj, opts...) +} + +func (c *labelWriterClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + // Ensure the patch doesn't remove the labels by applying the new patch. + latestObj := obj.DeepCopyObject().(client.Object) + if err := c.base.Get(ctx, client.ObjectKeyFromObject(obj), latestObj); err != nil { + return err + } + fakeClient := fake. + NewClientBuilder(). + WithRESTMapper(c.base.RESTMapper()). + WithScheme(c.base.Scheme()). + WithObjects(latestObj). + Build() + if err := fakeClient.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + return c.Update(ctx, obj) +} + +func (c *labelWriterClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return c.base.DeleteAllOf(ctx, obj, opts...) +} + +func (c *labelWriterClient) SubResource(subResource string) client.SubResourceWriter { + return c.base.SubResource(subResource) +} + +func (c *labelWriterClient) Status() client.SubResourceWriter { + return c.base.Status() +} + +func NewLabelWriterClient(c client.Client, labels map[string]string) DelegatingClient { + return newDelegatingWriteClient(c, &labelWriterClient{ + base: c, + labels: labels, + }) +} diff --git a/e2e/operator_context_test.go b/e2e/operator_context_test.go index 859801c692..1f4a7608dc 100644 --- a/e2e/operator_context_test.go +++ b/e2e/operator_context_test.go @@ -144,7 +144,7 @@ type OperatorContext struct { namespace, pubNamespace string - kClient client.Client + kClient DelegatingClient kubeClient kubernetes.Interface operatorClient clientset.Interface } @@ -175,18 +175,18 @@ func newOperatorContext(t *testing.T) *OperatorContext { T: t, namespace: namespace, pubNamespace: pubNamespace, - kClient: c, kubeClient: kubeClient, operatorClient: operatorClient, } + tctx.kClient = NewLabelWriterClient(c, tctx.getSubTestLabels()) t.Cleanup(func() { - if err := cleanupResources(ctx, kubeconfig, c, tctx.getSubTestLabelValue()); err != nil { + if err := cleanupResources(ctx, kubeconfig, tctx.kClient, tctx.getSubTestLabelValue()); err != nil { t.Fatalf("unable to cleanup resources: %s", err) } cancel() }) - if err := createBaseResources(ctx, c, namespace, pubNamespace, tctx.getSubTestLabels()); err != nil { + if err := createBaseResources(ctx, tctx.kClient, namespace, pubNamespace); err != nil { t.Fatalf("create resources: %s", err) } @@ -225,36 +225,34 @@ func (tctx *OperatorContext) getSubTestLabels() map[string]string { // createBaseResources creates resources the operator requires to exist already. // These are resources which don't depend on runtime state and can thus be deployed // statically, allowing to run the operator without critical write permissions. -func createBaseResources(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string, labels map[string]string) error { - if err := createNamespaces(ctx, kubeClient, opNamespace, publicNamespace, labels); err != nil { +func createBaseResources(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string) error { + if err := createNamespaces(ctx, kubeClient, opNamespace, publicNamespace); err != nil { return err } - if err := createGCPSecretResources(ctx, kubeClient, opNamespace, labels); err != nil { + if err := createGCPSecretResources(ctx, kubeClient, opNamespace); err != nil { return err } - if err := createCollectorResources(ctx, kubeClient, opNamespace, labels); err != nil { + if err := createCollectorResources(ctx, kubeClient, opNamespace); err != nil { return err } - if err := createAlertmanagerResources(ctx, kubeClient, opNamespace, labels); err != nil { + if err := createAlertmanagerResources(ctx, kubeClient, opNamespace); err != nil { return err } return nil } -func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string, labels map[string]string) error { +func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string) error { if err := kubeClient.Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: opNamespace, - Labels: labels, + Name: opNamespace, }, }); err != nil { return err } if err := kubeClient.Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: publicNamespace, - Labels: labels, + Name: publicNamespace, }, }); err != nil { return err @@ -262,7 +260,7 @@ func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace return nil } -func createGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error { +func createGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string) error { if gcpServiceAccount != "" { b, err := os.ReadFile(gcpServiceAccount) if err != nil { @@ -272,7 +270,6 @@ func createGCPSecretResources(ctx context.Context, kubeClient client.Client, nam ObjectMeta: metav1.ObjectMeta{ Name: "user-gcp-service-account", Namespace: namespace, - Labels: labels, }, Data: map[string][]byte{ "key.json": b, @@ -290,11 +287,10 @@ func parseResourceYAML(b []byte) (runtime.Object, error) { return obj, err } -func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error { +func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace string) error { if err := kubeClient.Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: operator.NameCollector, - Labels: labels, Namespace: namespace, }, }); err != nil { @@ -306,8 +302,7 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam if err := kubeClient.Create(ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName + ":" + namespace, - Labels: labels, + Name: clusterRoleName + ":" + namespace, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", @@ -336,12 +331,6 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam } collector := obj.(*appsv1.DaemonSet) collector.Namespace = namespace - if collector.Labels == nil { - collector.Labels = map[string]string{} - } - for k, v := range labels { - collector.Labels[k] = v - } if err = kubeClient.Create(ctx, collector); err != nil { return fmt.Errorf("create collector DaemonSet: %w", err) @@ -349,7 +338,7 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam return nil } -func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error { +func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, namespace string) error { evaluatorBytes, err := os.ReadFile(ruleEvalManifest) if err != nil { return fmt.Errorf("read rule-evaluator YAML: %w", err) @@ -361,12 +350,6 @@ func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, } evaluator := obj.(*appsv1.Deployment) evaluator.Namespace = namespace - if evaluator.Labels == nil { - evaluator.Labels = map[string]string{} - } - for k, v := range labels { - evaluator.Labels[k] = v - } if err := kubeClient.Create(ctx, evaluator); err != nil { return fmt.Errorf("create rule-evaluator Deployment: %w", err) @@ -387,14 +370,6 @@ func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, } obj.SetNamespace(namespace) - labels := obj.GetLabels() - if labels == nil { - labels = map[string]string{} - } - for k, v := range labels { - labels[k] = v - } - obj.SetLabels(labels) if err := kubeClient.Create(ctx, obj); err != nil { return fmt.Errorf("create object at index %d: %w", i, err) @@ -413,6 +388,7 @@ func (tctx *OperatorContext) subtest(f func(context.Context, *OperatorContext)) return func(t *testing.T) { childCtx := *tctx childCtx.T = t + childCtx.kClient = NewLabelWriterClient(tctx.kClient.Base(), tctx.getSubTestLabels()) f(context.TODO(), &childCtx) } }