Skip to content

Commit

Permalink
Automatically add subtest labels for cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
TheSpiritXIII committed Oct 2, 2023
1 parent 12d04a0 commit f80044d
Show file tree
Hide file tree
Showing 2 changed files with 211 additions and 41 deletions.
194 changes: 194 additions & 0 deletions e2e/kubeclient.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package e2e contains tests that validate the behavior of gmp-operator against a cluster.
// To make tests simple and fast, the test suite runs the operator internally. The CRDs
// are expected to be installed out of band (along with the operator deployment itself in
// a real world setup).
package e2e

import (
"context"

"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

type DelegatingClient interface {
client.Client
Base() client.Client
}

// delegatingWriteSubResourceClient delegates all mutating functions to a writer client.
type delegatingWriteSubResourceClient struct {
base client.SubResourceClient
writer client.SubResourceWriter
}

func (c *delegatingWriteSubResourceClient) Get(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error {
return c.base.Get(ctx, obj, subResource, opts...)
}

func (c *delegatingWriteSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error {
return c.writer.Create(ctx, obj, subResource, opts...)
}

func (c *delegatingWriteSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
return c.writer.Update(ctx, obj, opts...)
}

func (c *delegatingWriteSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
return c.writer.Patch(ctx, obj, patch, opts...)
}

// WriterClient comprises of all mutating functions in client.Client.
type WriterClient interface {
client.Writer
client.StatusClient
SubResource(subResource string) client.SubResourceWriter
}

// delegatingWriteClient delegates all mutating functions to a writer client.
type delegatingWriteClient struct {
base client.Client
writer WriterClient
}

func (c *delegatingWriteClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
return c.base.Get(ctx, key, obj, opts...)
}

func (c *delegatingWriteClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
return c.base.List(ctx, list, opts...)
}

func (c *delegatingWriteClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
return c.writer.Create(ctx, obj, opts...)
}

func (c *delegatingWriteClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
return c.writer.Delete(ctx, obj, opts...)
}

func (c *delegatingWriteClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
return c.writer.Update(ctx, obj, opts...)
}

func (c *delegatingWriteClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
return c.writer.Patch(ctx, obj, patch, opts...)
}

func (c *delegatingWriteClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
return c.writer.DeleteAllOf(ctx, obj, opts...)
}

func (c *delegatingWriteClient) SubResource(subResource string) client.SubResourceClient {
return &delegatingWriteSubResourceClient{
base: c.base.SubResource(subResource),
writer: c.writer.SubResource(subResource),
}
}

func (c *delegatingWriteClient) Status() client.SubResourceWriter {
return c.writer.Status()
}

func (c *delegatingWriteClient) Scheme() *runtime.Scheme {
return c.base.Scheme()
}

func (c *delegatingWriteClient) RESTMapper() meta.RESTMapper {
return c.base.RESTMapper()
}

func (c *delegatingWriteClient) Base() client.Client {
return c.base
}

func newDelegatingWriteClient(c client.Client, w WriterClient) DelegatingClient {
return &delegatingWriteClient{
base: c,
writer: w,
}
}

// labelWriterClient adds common labels to all objects mutated by this client.
type labelWriterClient struct {
base client.Client
labels map[string]string
}

func (c *labelWriterClient) setLabels(obj client.Object) {
labels := obj.GetLabels()
if labels == nil {
labels = map[string]string{}
}
for k, v := range c.labels {
labels[k] = v
}
obj.SetLabels(labels)
}

func (c *labelWriterClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
c.setLabels(obj)
return c.base.Create(ctx, obj, opts...)
}

func (c *labelWriterClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
return c.base.Delete(ctx, obj, opts...)
}

func (c *labelWriterClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
c.setLabels(obj)
return c.base.Update(ctx, obj, opts...)
}

func (c *labelWriterClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
// Ensure the patch doesn't remove the labels by applying the new patch.
latestObj := obj.DeepCopyObject().(client.Object)
if err := c.base.Get(ctx, client.ObjectKeyFromObject(obj), latestObj); err != nil {
return err
}
fakeClient := fake.
NewClientBuilder().
WithRESTMapper(c.base.RESTMapper()).
WithScheme(c.base.Scheme()).
WithObjects(latestObj).
Build()
if err := fakeClient.Patch(ctx, obj, patch, opts...); err != nil {
return err
}
return c.Update(ctx, obj)
}

func (c *labelWriterClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
return c.base.DeleteAllOf(ctx, obj, opts...)
}

func (c *labelWriterClient) SubResource(subResource string) client.SubResourceWriter {
return c.base.SubResource(subResource)
}

func (c *labelWriterClient) Status() client.SubResourceWriter {
return c.base.Status()
}

func NewLabelWriterClient(c client.Client, labels map[string]string) DelegatingClient {
return newDelegatingWriteClient(c, &labelWriterClient{
base: c,
labels: labels,
})
}
58 changes: 17 additions & 41 deletions e2e/operator_context_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ type OperatorContext struct {

namespace, pubNamespace string

kClient client.Client
kClient DelegatingClient
kubeClient kubernetes.Interface
operatorClient clientset.Interface
}
Expand Down Expand Up @@ -175,18 +175,18 @@ func newOperatorContext(t *testing.T) *OperatorContext {
T: t,
namespace: namespace,
pubNamespace: pubNamespace,
kClient: c,
kubeClient: kubeClient,
operatorClient: operatorClient,
}
tctx.kClient = NewLabelWriterClient(c, tctx.getSubTestLabels())
t.Cleanup(func() {
if err := cleanupResources(ctx, kubeconfig, c, tctx.getSubTestLabelValue()); err != nil {
if err := cleanupResources(ctx, kubeconfig, tctx.kClient, tctx.getSubTestLabelValue()); err != nil {
t.Fatalf("unable to cleanup resources: %s", err)
}
cancel()
})

if err := createBaseResources(ctx, c, namespace, pubNamespace, tctx.getSubTestLabels()); err != nil {
if err := createBaseResources(ctx, tctx.kClient, namespace, pubNamespace); err != nil {
t.Fatalf("create resources: %s", err)
}

Expand Down Expand Up @@ -225,44 +225,42 @@ func (tctx *OperatorContext) getSubTestLabels() map[string]string {
// createBaseResources creates resources the operator requires to exist already.
// These are resources which don't depend on runtime state and can thus be deployed
// statically, allowing to run the operator without critical write permissions.
func createBaseResources(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string, labels map[string]string) error {
if err := createNamespaces(ctx, kubeClient, opNamespace, publicNamespace, labels); err != nil {
func createBaseResources(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string) error {
if err := createNamespaces(ctx, kubeClient, opNamespace, publicNamespace); err != nil {
return err
}

if err := createGCPSecretResources(ctx, kubeClient, opNamespace, labels); err != nil {
if err := createGCPSecretResources(ctx, kubeClient, opNamespace); err != nil {
return err
}
if err := createCollectorResources(ctx, kubeClient, opNamespace, labels); err != nil {
if err := createCollectorResources(ctx, kubeClient, opNamespace); err != nil {
return err
}
if err := createAlertmanagerResources(ctx, kubeClient, opNamespace, labels); err != nil {
if err := createAlertmanagerResources(ctx, kubeClient, opNamespace); err != nil {
return err
}
return nil
}

func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string, labels map[string]string) error {
func createNamespaces(ctx context.Context, kubeClient client.Client, opNamespace, publicNamespace string) error {
if err := kubeClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: opNamespace,
Labels: labels,
Name: opNamespace,
},
}); err != nil {
return err
}
if err := kubeClient.Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: publicNamespace,
Labels: labels,
Name: publicNamespace,
},
}); err != nil {
return err
}
return nil
}

func createGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error {
func createGCPSecretResources(ctx context.Context, kubeClient client.Client, namespace string) error {
if gcpServiceAccount != "" {
b, err := os.ReadFile(gcpServiceAccount)
if err != nil {
Expand All @@ -272,7 +270,6 @@ func createGCPSecretResources(ctx context.Context, kubeClient client.Client, nam
ObjectMeta: metav1.ObjectMeta{
Name: "user-gcp-service-account",
Namespace: namespace,
Labels: labels,
},
Data: map[string][]byte{
"key.json": b,
Expand All @@ -290,11 +287,10 @@ func parseResourceYAML(b []byte) (runtime.Object, error) {
return obj, err
}

func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error {
func createCollectorResources(ctx context.Context, kubeClient client.Client, namespace string) error {
if err := kubeClient.Create(ctx, &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: operator.NameCollector,
Labels: labels,
Namespace: namespace,
},
}); err != nil {
Expand All @@ -306,8 +302,7 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam

if err := kubeClient.Create(ctx, &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName + ":" + namespace,
Labels: labels,
Name: clusterRoleName + ":" + namespace,
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Expand Down Expand Up @@ -336,20 +331,14 @@ func createCollectorResources(ctx context.Context, kubeClient client.Client, nam
}
collector := obj.(*appsv1.DaemonSet)
collector.Namespace = namespace
if collector.Labels == nil {
collector.Labels = map[string]string{}
}
for k, v := range labels {
collector.Labels[k] = v
}

if err = kubeClient.Create(ctx, collector); err != nil {
return fmt.Errorf("create collector DaemonSet: %w", err)
}
return nil
}

func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, namespace string, labels map[string]string) error {
func createAlertmanagerResources(ctx context.Context, kubeClient client.Client, namespace string) error {
evaluatorBytes, err := os.ReadFile(ruleEvalManifest)
if err != nil {
return fmt.Errorf("read rule-evaluator YAML: %w", err)
Expand All @@ -361,12 +350,6 @@ func createAlertmanagerResources(ctx context.Context, kubeClient client.Client,
}
evaluator := obj.(*appsv1.Deployment)
evaluator.Namespace = namespace
if evaluator.Labels == nil {
evaluator.Labels = map[string]string{}
}
for k, v := range labels {
evaluator.Labels[k] = v
}

if err := kubeClient.Create(ctx, evaluator); err != nil {
return fmt.Errorf("create rule-evaluator Deployment: %w", err)
Expand All @@ -387,14 +370,6 @@ func createAlertmanagerResources(ctx context.Context, kubeClient client.Client,
}

obj.SetNamespace(namespace)
labels := obj.GetLabels()
if labels == nil {
labels = map[string]string{}
}
for k, v := range labels {
labels[k] = v
}
obj.SetLabels(labels)

if err := kubeClient.Create(ctx, obj); err != nil {
return fmt.Errorf("create object at index %d: %w", i, err)
Expand All @@ -413,6 +388,7 @@ func (tctx *OperatorContext) subtest(f func(context.Context, *OperatorContext))
return func(t *testing.T) {
childCtx := *tctx
childCtx.T = t
childCtx.kClient = NewLabelWriterClient(tctx.kClient.Base(), tctx.getSubTestLabels())
f(context.TODO(), &childCtx)
}
}
Expand Down

0 comments on commit f80044d

Please sign in to comment.