From e7c42ef8a04d883edd56dc291bee40db5d29ec6d Mon Sep 17 00:00:00 2001 From: Shawn Hurley Date: Fri, 22 Apr 2022 15:00:50 -0400 Subject: [PATCH] Adding e2e tests for new plugins --- pkg/admission/mutatingwebhook/plugin.go | 4 +- pkg/admission/plugins.go | 17 +- pkg/admission/validatingwebhook/plugin.go | 2 +- pkg/admission/webhook/generic_webhook.go | 20 +- pkg/admission/webhook/generic_webhook_test.go | 45 +-- .../e2e/apibinding/apibinding_webhook_test.go | 333 ++++++++++++++++++ test/e2e/conformance/webhook_test.go | 226 ++++++------ test/e2e/fixtures/webhook/webhook.go | 160 +++++++++ 8 files changed, 658 insertions(+), 149 deletions(-) create mode 100644 test/e2e/apibinding/apibinding_webhook_test.go create mode 100644 test/e2e/fixtures/webhook/webhook.go diff --git a/pkg/admission/mutatingwebhook/plugin.go b/pkg/admission/mutatingwebhook/plugin.go index 6f2c2b342c69..c0233b643907 100644 --- a/pkg/admission/mutatingwebhook/plugin.go +++ b/pkg/admission/mutatingwebhook/plugin.go @@ -101,11 +101,11 @@ func Register(plugins *admission.Plugins) { } func (a *Plugin) Admit(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) error { - return a.Dispatch(ctx, attr, o) + return a.WebhookDispatcher.Dispatch(ctx, attr, o) } // SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. func (p *Plugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { - p.WebhookDispatcher.SetExternalKubeInformerFactory(f) + p.WebhookDispatcher.SetHookSource(configuration.NewMutatingWebhookConfigurationManager(f)) p.Plugin.SetExternalKubeInformerFactory(f) } diff --git a/pkg/admission/plugins.go b/pkg/admission/plugins.go index b2dd953e06b0..99cac07ba2e1 100644 --- a/pkg/admission/plugins.go +++ b/pkg/admission/plugins.go @@ -22,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle" "k8s.io/apiserver/pkg/admission/plugin/resourcequota" mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating" + validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating" kubeapiserveroptions "k8s.io/kubernetes/pkg/kubeapiserver/options" certapproval "k8s.io/kubernetes/plugin/pkg/admission/certificates/approval" certsigning "k8s.io/kubernetes/plugin/pkg/admission/certificates/signing" @@ -110,14 +111,14 @@ var defaultOnPluginsInKcp = sets.NewString( // new plugins got added upstream and to react (enable or disable by default). We // have a unit test in place to avoid drift. var defaultOnKubePluginsInKube = sets.NewString( - lifecycle.PluginName, // NamespaceLifecycle - limitranger.PluginName, // LimitRanger - serviceaccount.PluginName, // ServiceAccount - setdefault.PluginName, // DefaultStorageClass - resize.PluginName, // PersistentVolumeClaimResize - defaulttolerationseconds.PluginName, // DefaultTolerationSeconds - //mutatingwebhook.PluginName, // MutatingAdmissionWebhook - //validatingwebhook.PluginName, // ValidatingAdmissionWebhook + lifecycle.PluginName, // NamespaceLifecycle + limitranger.PluginName, // LimitRanger + serviceaccount.PluginName, // ServiceAccount + setdefault.PluginName, // DefaultStorageClass + resize.PluginName, // PersistentVolumeClaimResize + defaulttolerationseconds.PluginName, // DefaultTolerationSeconds + mutatingwebhook.PluginName, // MutatingAdmissionWebhook + validatingwebhook.PluginName, // ValidatingAdmissionWebhook resourcequota.PluginName, // ResourceQuota storageobjectinuseprotection.PluginName, // StorageObjectInUseProtection podpriority.PluginName, // PodPriority diff --git a/pkg/admission/validatingwebhook/plugin.go b/pkg/admission/validatingwebhook/plugin.go index 6a18c1f2d1ce..c91ebc37e0db 100644 --- a/pkg/admission/validatingwebhook/plugin.go +++ b/pkg/admission/validatingwebhook/plugin.go @@ -104,6 +104,6 @@ func (a *Plugin) Validate(ctx context.Context, attr admission.Attributes, o admi // SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. func (p *Plugin) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { - p.WebhookDispatcher.SetExternalKubeInformerFactory(f) + p.WebhookDispatcher.SetHookSource(configuration.NewValidatingWebhookConfigurationManager(f)) p.Plugin.SetExternalKubeInformerFactory(f) } diff --git a/pkg/admission/webhook/generic_webhook.go b/pkg/admission/webhook/generic_webhook.go index 4252991ae4a0..23a1f30ef7c2 100644 --- a/pkg/admission/webhook/generic_webhook.go +++ b/pkg/admission/webhook/generic_webhook.go @@ -26,12 +26,10 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/admission/configuration" "k8s.io/apiserver/pkg/admission/plugin/webhook" "k8s.io/apiserver/pkg/admission/plugin/webhook/generic" "k8s.io/apiserver/pkg/admission/plugin/webhook/rules" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/client-go/informers" ) type WebhookDispatcher struct { @@ -70,7 +68,9 @@ func (p *WebhookDispatcher) Dispatch(ctx context.Context, attr admission.Attribu // Determine the type of request, is it api binding or not. if workspace, isAPIBinding := p.getAPIBindingWorkspace(attr, lcluster); isAPIBinding { whAccessor = p.restrictToLogicalCluster(hooks, workspace) + fmt.Printf("\n\nworkspace: %v\nwhAccessor: %v\nhooks:%v", workspace, whAccessor, hooks) } else { + fmt.Printf("here") whAccessor = p.restrictToLogicalCluster(hooks, lcluster) } @@ -80,18 +80,27 @@ func (p *WebhookDispatcher) Dispatch(ctx context.Context, attr admission.Attribu func (p *WebhookDispatcher) getAPIBindingWorkspace(attr admission.Attributes, lc logicalcluster.LogicalCluster) (logicalcluster.LogicalCluster, bool) { l, err := p.apiBinding.List(labels.Everything()) if err != nil { + fmt.Printf("here1") return logicalcluster.New(""), false } for _, apiBinding := range l { if apiBinding.GetClusterName() != lc.String() { + fmt.Printf("apiBinding: %v lc: %v", apiBinding.GetClusterName(), lc.String()) continue } for _, br := range apiBinding.Status.BoundResources { if br.Group == attr.GetResource().Group && br.Resource == attr.GetResource().Resource { - return logicalcluster.New(apiBinding.Status.BoundAPIExport.Workspace.WorkspaceName), true + p, hasParent := logicalcluster.From(apiBinding).Parent() + if !hasParent { + fmt.Printf("here3") + return logicalcluster.New(""), false + } + fmt.Printf("here4") + return p.Join(apiBinding.Status.BoundAPIExport.Workspace.WorkspaceName), true } } } + fmt.Printf("here5") return logicalcluster.New(""), false } @@ -107,9 +116,8 @@ func (p *WebhookDispatcher) restrictToLogicalCluster(hooks []webhook.WebhookAcce return wh } -// SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. -func (p *WebhookDispatcher) SetExternalKubeInformerFactory(f informers.SharedInformerFactory) { - p.hookSource = configuration.NewValidatingWebhookConfigurationManager(f) +func (p *WebhookDispatcher) SetHookSource(s generic.Source) { + p.hookSource = s } // SetExternalKubeInformerFactory implements the WantsExternalKubeInformerFactory interface. diff --git a/pkg/admission/webhook/generic_webhook_test.go b/pkg/admission/webhook/generic_webhook_test.go index 69d3a3921873..840389c31621 100644 --- a/pkg/admission/webhook/generic_webhook_test.go +++ b/pkg/admission/webhook/generic_webhook_test.go @@ -57,18 +57,23 @@ type validatingDispatcher struct { } func (d *validatingDispatcher) Dispatch(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces, hooks []webhook.WebhookAccessor) error { + fmt.Printf("\nexpected: %v", d.hooks) + fmt.Printf("\ngot: %v", hooks) if len(hooks) != len(d.hooks) { return fmt.Errorf("invalid number of hooks sent to dispatcher") } uidMatches := map[string]*struct{}{} for _, h := range hooks { + fmt.Printf("\n%v", h.GetUID()) for _, expectedHook := range d.hooks { + fmt.Printf("\n%v", expectedHook.GetUID()) if h.GetUID() == expectedHook.GetUID() { uidMatches[h.GetUID()] = &struct{}{} } } } if len(uidMatches) != len(d.hooks) { + fmt.Printf("%#v", uidMatches) return fmt.Errorf("hooks UID did not match expected") } return nil @@ -127,18 +132,18 @@ func TestDispatch(t *testing.T) { "cowboys", admission.Create, ), - cluster: "dest-cluster", + cluster: "root:org:dest-cluster", expectedHooks: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("1", "api-registration-hook", logicalcluster.New("source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("1", "api-registration-hook", logicalcluster.New("root:org:source-cluster"), nil), }, hooksInSource: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("1", "api-registration-hook", logicalcluster.New("source-cluster"), nil), - webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("dest-cluster"), nil), + webhook.NewValidatingWebhookAccessor("1", "api-registration-hook", logicalcluster.New("root:org:source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("root:org:dest-cluster"), nil), }, apiBindings: []*v1alpha1.APIBinding{ { ObjectMeta: metav1.ObjectMeta{ - ClusterName: "dest-cluster", + ClusterName: "root:org:dest-cluster", }, Status: v1alpha1.APIBindingStatus{ BoundResources: []v1alpha1.BoundAPIResource{ @@ -164,14 +169,14 @@ func TestDispatch(t *testing.T) { "cowboys", admission.Create, ), - cluster: "dest-cluster", + cluster: "root:org:dest-cluster", expectedHooks: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("dest-cluster"), nil), + webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("root:org:dest-cluster"), nil), }, hooksInSource: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("1", "cowboy-hook", logicalcluster.New("source-cluster"), nil), - webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("source-cluster"), nil), - webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("dest-cluster"), nil), + webhook.NewValidatingWebhookAccessor("1", "cowboy-hook", logicalcluster.New("root:org:source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("root:org:source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("root:org:dest-cluster"), nil), }, }, { @@ -182,19 +187,19 @@ func TestDispatch(t *testing.T) { "cowboys", admission.Create, ), - cluster: "dest-cluster", + cluster: "root:org:dest-cluster", expectedHooks: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("dest-cluster"), nil), + webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("root:org:dest-cluster"), nil), }, hooksInSource: []webhook.WebhookAccessor{ - webhook.NewValidatingWebhookAccessor("1", "cowboy-hook", logicalcluster.New("source-cluster"), nil), - webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("source-cluster"), nil), - webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("dest-cluster"), nil), + webhook.NewValidatingWebhookAccessor("1", "cowboy-hook", logicalcluster.New("root:org:source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("2", "secrets", logicalcluster.New("root:org:source-cluster"), nil), + webhook.NewValidatingWebhookAccessor("3", "secrets", logicalcluster.New("root:org:dest-cluster"), nil), }, apiBindings: []*v1alpha1.APIBinding{ { ObjectMeta: metav1.ObjectMeta{ - ClusterName: "dest-cluster", + ClusterName: "root:org:dest-cluster", }, Status: v1alpha1.APIBindingStatus{ BoundResources: []v1alpha1.BoundAPIResource{ @@ -212,7 +217,7 @@ func TestDispatch(t *testing.T) { }, { ObjectMeta: metav1.ObjectMeta{ - ClusterName: "dest-cluster-2", + ClusterName: "root:org:dest-cluster-2", }, Status: v1alpha1.APIBindingStatus{ BoundResources: []v1alpha1.BoundAPIResource{ @@ -238,7 +243,7 @@ func TestDispatch(t *testing.T) { "cowboys", admission.Create, ), - cluster: "dest-cluster", + cluster: "root:org:dest-cluster", apiBindingsSynced: func() bool { return false }, @@ -252,7 +257,7 @@ func TestDispatch(t *testing.T) { "cowboys", admission.Create, ), - cluster: "dest-cluster", + cluster: "root:org:dest-cluster", hookSourceNotSynced: true, wantErr: true, }, @@ -278,7 +283,7 @@ func TestDispatch(t *testing.T) { ctx := request.WithCluster(context.Background(), request.Cluster{Name: logicalcluster.New(tc.cluster)}) if err := o.Dispatch(ctx, tc.attr, nil); (err != nil) != tc.wantErr { - t.Fatalf("Validate() error = %v, wantErr %v", err, tc.wantErr) + t.Fatalf("Dispatch() error = %v, wantErr %v", err, tc.wantErr) } }) } diff --git a/test/e2e/apibinding/apibinding_webhook_test.go b/test/e2e/apibinding/apibinding_webhook_test.go new file mode 100644 index 000000000000..384388719fa9 --- /dev/null +++ b/test/e2e/apibinding/apibinding_webhook_test.go @@ -0,0 +1,333 @@ +/* +Copyright 2021 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apibinding + +import ( + "context" + "sync" + "testing" + "time" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + + "github.com/kcp-dev/kcp/config/helpers" + apisv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/apis/v1alpha1" + clientset "github.com/kcp-dev/kcp/pkg/client/clientset/versioned" + webhookserver "github.com/kcp-dev/kcp/test/e2e/fixtures/webhook" + "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest/v1alpha1" + client "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/client/clientset/versioned" + "github.com/kcp-dev/kcp/test/e2e/framework" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/restmapper" +) + +func TestAPIBindingMutatingWebhook(t *testing.T) { + t.Parallel() + + server := framework.SharedKcpServer(t) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + orgClusterName := framework.NewOrganizationFixture(t, server) + sourceWorkspace := framework.NewWorkspaceFixture(t, server, orgClusterName, "Universal") + targetWorkspace := framework.NewWorkspaceFixture(t, server, orgClusterName, "Universal") + + cfg := server.DefaultConfig(t) + + kcpClients, err := clientset.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct kcp cluster client for server") + + dynamicClients, err := dynamic.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct dynamic cluster client for server") + + kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + + t.Logf("Install a cowboys APIResourceSchema into workspace %q", sourceWorkspace) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(kcpClients.Cluster(sourceWorkspace).Discovery())) + err = helpers.CreateResourceFromFS(ctx, dynamicClients.Cluster(sourceWorkspace), mapper, "apiresourceschema_cowboys.yaml", testFiles) + require.NoError(t, err) + + t.Logf("Create an APIExport for it") + cowboysAPIExport := &apisv1alpha1.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "today-cowboys", + }, + Spec: apisv1alpha1.APIExportSpec{ + LatestResourceSchemas: []string{"today.cowboys.wildwest.dev"}, + }, + } + _, err = kcpClients.Cluster(sourceWorkspace).ApisV1alpha1().APIExports().Create(ctx, cowboysAPIExport, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create an APIBinding in workspace %q that points to the today-cowboys export", targetWorkspace) + require.NoError(t, err) + apiBinding := &apisv1alpha1.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cowboys", + }, + Spec: apisv1alpha1.APIBindingSpec{ + Reference: apisv1alpha1.ExportReference{ + Workspace: &apisv1alpha1.WorkspaceExportReference{ + WorkspaceName: sourceWorkspace.Base(), + ExportName: cowboysAPIExport.Name, + }, + }, + }, + } + + _, err = kcpClients.Cluster(targetWorkspace).ApisV1alpha1().APIBindings().Create(ctx, apiBinding, metav1.CreateOptions{}) + require.NoError(t, err) + + scheme := runtime.NewScheme() + err = admissionregistrationv1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission registration v1 scheme") + err = v1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission v1 scheme") + err = v1alpha1.AddToScheme(scheme) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") + cowbyClients, err := client.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") + codecs := serializer.NewCodecFactory(scheme) + deserializer := codecs.UniversalDeserializer() + + //Create Test Server and Create Validating Webhook for Cowboys in source cluster. + testWebhook := webhookserver.WebhookServer{ + Response: v1.AdmissionResponse{ + Allowed: true, + }, + ObjectGVK: schema.GroupVersionKind{ + Group: "wildwest.dev", + Version: "v1alpha1", + Kind: "Cowboy", + }, + T: t, + Lock: sync.Mutex{}, + Deserializer: deserializer, + } + + port, err := framework.GetFreePort(t) + require.NoError(t, err, "failed to get free port for test webhook") + testWebhook.StartServer(ctx, server, port) + + t.Logf("Installing webhook into the source workspace") + sideEffect := admissionregistrationv1.SideEffectClassNone + webhook := &admissionregistrationv1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test-webhook"}, + Webhooks: []admissionregistrationv1.MutatingWebhook{{ + Name: "test-webhook.cowboy.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: testWebhook.GetURL(), + CABundle: cfg.CAData, + }, + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"wildwest.dev"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"cowboys"}, + }, + }}, + SideEffects: &sideEffect, + AdmissionReviewVersions: []string{"v1"}, + }}, + } + _, err = kubeClusterClient.Cluster(sourceWorkspace).AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) + require.NoError(t, err, "failed to add validating webhook configurations") + + cowboy := v1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing", + }, + Spec: v1alpha1.CowboySpec{}, + } + + // Avoid race condition here by making sure that CRD is served after installing the types into logical clusters + t.Logf("Creating cowboy resource in target logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(targetWorkspace).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + //Avoid race condition where webhook informer is not updated before the call to create was made. + t.Log("Verify webhook is eventually called") + require.Eventually(t, func() bool { + return testWebhook.Calls == 1 + }, wait.ForeverTestTimeout, 100*time.Millisecond) +} + +func TestAPIBindingValidatingWebhook(t *testing.T) { + t.Parallel() + + server := framework.SharedKcpServer(t) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + orgClusterName := framework.NewOrganizationFixture(t, server) + sourceWorkspace := framework.NewWorkspaceFixture(t, server, orgClusterName, "Universal") + targetWorkspace := framework.NewWorkspaceFixture(t, server, orgClusterName, "Universal") + + cfg := server.DefaultConfig(t) + + kcpClients, err := clientset.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct kcp cluster client for server") + + dynamicClients, err := dynamic.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct dynamic cluster client for server") + + kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + + t.Logf("Install a cowboys APIResourceSchema into workspace %q", sourceWorkspace) + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(kcpClients.Cluster(sourceWorkspace).Discovery())) + err = helpers.CreateResourceFromFS(ctx, dynamicClients.Cluster(sourceWorkspace), mapper, "apiresourceschema_cowboys.yaml", testFiles) + require.NoError(t, err) + + t.Logf("Create an APIExport for it") + cowboysAPIExport := &apisv1alpha1.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "today-cowboys", + }, + Spec: apisv1alpha1.APIExportSpec{ + LatestResourceSchemas: []string{"today.cowboys.wildwest.dev"}, + }, + } + _, err = kcpClients.Cluster(sourceWorkspace).ApisV1alpha1().APIExports().Create(ctx, cowboysAPIExport, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Logf("Create an APIBinding in workspace %q that points to the today-cowboys export", targetWorkspace) + require.NoError(t, err) + apiBinding := &apisv1alpha1.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cowboys", + }, + Spec: apisv1alpha1.APIBindingSpec{ + Reference: apisv1alpha1.ExportReference{ + Workspace: &apisv1alpha1.WorkspaceExportReference{ + WorkspaceName: sourceWorkspace.Base(), + ExportName: cowboysAPIExport.Name, + }, + }, + }, + } + + _, err = kcpClients.Cluster(targetWorkspace).ApisV1alpha1().APIBindings().Create(ctx, apiBinding, metav1.CreateOptions{}) + require.NoError(t, err) + + scheme := runtime.NewScheme() + err = admissionregistrationv1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission registration v1 scheme") + err = v1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission v1 scheme") + err = v1alpha1.AddToScheme(scheme) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") + cowbyClients, err := client.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") + codecs := serializer.NewCodecFactory(scheme) + deserializer := codecs.UniversalDeserializer() + + //Create Test Server and Create Validating Webhook for Cowboys in source cluster. + testWebhook := webhookserver.WebhookServer{ + Response: v1.AdmissionResponse{ + Allowed: true, + }, + ObjectGVK: schema.GroupVersionKind{ + Group: "wildwest.dev", + Version: "v1alpha1", + Kind: "Cowboy", + }, + T: t, + Lock: sync.Mutex{}, + Deserializer: deserializer, + } + + port, err := framework.GetFreePort(t) + require.NoError(t, err, "failed to get free port for test webhook") + testWebhook.StartServer(ctx, server, port) + + t.Logf("Installing webhook into the source workspace") + sideEffect := admissionregistrationv1.SideEffectClassNone + webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: "test-webhook.cowboy.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: testWebhook.GetURL(), + CABundle: cfg.CAData, + }, + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"wildwest.dev"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"cowboys"}, + }, + }}, + SideEffects: &sideEffect, + AdmissionReviewVersions: []string{"v1"}, + }}, + } + _, err = kubeClusterClient.Cluster(sourceWorkspace).AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) + require.NoError(t, err, "failed to add validating webhook configurations") + + cowboy := v1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing", + }, + Spec: v1alpha1.CowboySpec{}, + } + + // Avoid race condition here by making sure that CRD is served after installing the types into logical clusters + t.Logf("Creating cowboy resource in target logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(targetWorkspace).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + //Avoid race condition where webhook informer is not updated before the call to create was made. + t.Log("Verify webhook is eventually called") + require.Eventually(t, func() bool { + return testWebhook.Calls == 1 + }, wait.ForeverTestTimeout, 100*time.Millisecond) + +} diff --git a/test/e2e/conformance/webhook_test.go b/test/e2e/conformance/webhook_test.go index 0096b7225446..7ff27c4f2bd1 100644 --- a/test/e2e/conformance/webhook_test.go +++ b/test/e2e/conformance/webhook_test.go @@ -18,11 +18,6 @@ package conformance import ( "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "path/filepath" "sync" "testing" "time" @@ -40,27 +35,24 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" + webhookserver "github.com/kcp-dev/kcp/test/e2e/fixtures/webhook" "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest" "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest/v1alpha1" client "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/client/clientset/versioned" "github.com/kcp-dev/kcp/test/e2e/framework" ) -func TestWebhookInWorkspace(t *testing.T) { +func TestMutatingWebhookInWorkspace(t *testing.T) { t.Parallel() server := framework.SharedKcpServer(t) - dirPath := filepath.Dir(server.KubeconfigPath()) ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(cancelFunc) // using known path to cert and key cfg := server.DefaultConfig(t) - cfg.CertFile = filepath.Join(dirPath, "apiserver.crt") - cfg.KeyFile = filepath.Join(dirPath, "apiserver.key") scheme := runtime.NewScheme() err := admissionregistrationv1.AddToScheme(scheme) @@ -73,7 +65,7 @@ func TestWebhookInWorkspace(t *testing.T) { codecs := serializer.NewCodecFactory(scheme) deserializer := codecs.UniversalDeserializer() - testWebhook := testWebhookServer{ + testWebhook := webhookserver.WebhookServer{ Response: v1.AdmissionResponse{ Allowed: true, }, @@ -89,7 +81,7 @@ func TestWebhookInWorkspace(t *testing.T) { port, err := framework.GetFreePort(t) require.NoError(t, err, "failed to get free port for test webhook") - testWebhook.StartServer(ctx, cfg, port) + testWebhook.StartServer(ctx, server, port) organization := framework.NewOrganizationFixture(t, server) logicalClusters := []logicalcluster.LogicalCluster{ @@ -112,15 +104,14 @@ func TestWebhookInWorkspace(t *testing.T) { } t.Logf("Installing webhook into the first workspace") - url := fmt.Sprintf("https://localhost:%v/hello", port) sideEffect := admissionregistrationv1.SideEffectClassNone - webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{ + webhook := &admissionregistrationv1.MutatingWebhookConfiguration{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "test-webhook"}, - Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Webhooks: []admissionregistrationv1.MutatingWebhook{{ Name: "test-webhook.cowboy.io", ClientConfig: admissionregistrationv1.WebhookClientConfig{ - URL: &url, + URL: testWebhook.GetURL(), CABundle: cfg.CAData, }, Rules: []admissionregistrationv1.RuleWithOperations{{ @@ -137,7 +128,7 @@ func TestWebhookInWorkspace(t *testing.T) { AdmissionReviewVersions: []string{"v1"}, }}, } - _, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) + _, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) require.NoError(t, err, "failed to add validating webhook configurations") cowboy := v1alpha1.Cowboy{ @@ -176,115 +167,126 @@ func TestWebhookInWorkspace(t *testing.T) { }, wait.ForeverTestTimeout, 100*time.Millisecond) } -type testWebhookServer struct { - Response v1.AdmissionResponse - ObjectGVK schema.GroupVersionKind - T *testing.T - Deserializer runtime.Decoder +func TestValidatingWebhookInWorkspace(t *testing.T) { + t.Parallel() - Lock sync.Mutex - Calls int -} + server := framework.SharedKcpServer(t) -func (t *testWebhookServer) StartServer(ctx context.Context, cfg *rest.Config, port string) { - serv := &http.Server{Addr: fmt.Sprintf(":%v", port), Handler: t} - go func() { - <-ctx.Done() - t.T.Logf("Shutting down the HTTP server") - err := serv.Shutdown(context.TODO()) - if err != nil { - t.T.Errorf("failure to shutdown server: %v", err) - } - }() - go func() { - err := serv.ListenAndServeTLS(cfg.CertFile, cfg.KeyFile) - if err != nil && err != http.ErrServerClosed { - t.T.Error(err) - } - }() -} + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) -func (t *testWebhookServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { - // Make sure that this is a request for the object that was set. - if req.Body == nil { - msg := "Expected request body to be non-empty" - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - } + // using known path to cert and key + cfg := server.DefaultConfig(t) - data, err := ioutil.ReadAll(req.Body) - if err != nil { - msg := fmt.Sprintf("Request could not be decoded: %v", err) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - } + scheme := runtime.NewScheme() + err := admissionregistrationv1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission registration v1 scheme") + err = v1.AddToScheme(scheme) + require.NoError(t, err, "failed to add admission v1 scheme") + err = v1alpha1.AddToScheme(scheme) + require.NoError(t, err, "failed to add cowboy v1alpha1 to scheme") - // verify the content type is accurate - contentType := req.Header.Get("Content-Type") - if contentType != "application/json" { - msg := fmt.Sprintf("contentType=%s, expect application/json", contentType) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return - } + codecs := serializer.NewCodecFactory(scheme) + deserializer := codecs.UniversalDeserializer() - obj, gvk, err := t.Deserializer.Decode(data, nil, nil) - if err != nil { - msg := fmt.Sprintf("Unable to decode object: %v", err) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return + testWebhook := webhookserver.WebhookServer{ + Response: v1.AdmissionResponse{ + Allowed: true, + }, + ObjectGVK: schema.GroupVersionKind{ + Group: "wildwest.dev", + Version: "v1alpha1", + Kind: "Cowboy", + }, + T: t, + Lock: sync.Mutex{}, + Deserializer: deserializer, } - if *gvk != v1.SchemeGroupVersion.WithKind("AdmissionReview") { - msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return - } - requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) - if !ok { - //return an error - msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return - } - obj, objGVK, err := t.Deserializer.Decode(requestedAdmissionReview.Request.Object.Raw, nil, nil) - if err != nil { - msg := fmt.Sprintf("Unable to decode admissions reqeusted object: %v", err) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return + port, err := framework.GetFreePort(t) + require.NoError(t, err, "failed to get free port for test webhook") + testWebhook.StartServer(ctx, server, port) + + organization := framework.NewOrganizationFixture(t, server) + logicalClusters := []logicalcluster.LogicalCluster{ + framework.NewWorkspaceFixture(t, server, organization, "Universal"), + framework.NewWorkspaceFixture(t, server, organization, "Universal"), } - if t.ObjectGVK != *objGVK { - //return an error - msg := fmt.Sprintf("Expected ObjectGVK: %v but got: %T", t.ObjectGVK, obj) - t.T.Logf("%v", msg) - http.Error(resp, msg, http.StatusBadRequest) - return + kubeClusterClient, err := kubernetes.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct client for server") + cowbyClients, err := client.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct cowboy client for server") + apiExtensionsClients, err := apiextensionsclient.NewClusterForConfig(cfg) + require.NoError(t, err, "failed to construct apiextensions client for server") + + t.Logf("Install the Cowboy resources into logical clusters") + for _, logicalCluster := range logicalClusters { + t.Logf("Bootstrapping ClusterWorkspace CRDs in logical cluster %s", logicalCluster) + crdClient := apiExtensionsClients.Cluster(logicalCluster).ApiextensionsV1().CustomResourceDefinitions() + wildwest.Create(t, crdClient, metav1.GroupResource{Group: "wildwest.dev", Resource: "cowboys"}) } - responseAdmissionReview := &v1.AdmissionReview{ - TypeMeta: requestedAdmissionReview.TypeMeta, + t.Logf("Installing webhook into the first workspace") + sideEffect := admissionregistrationv1.SideEffectClassNone + webhook := &admissionregistrationv1.ValidatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + Name: "test-webhook.cowboy.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: testWebhook.GetURL(), + CABundle: cfg.CAData, + }, + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + }, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"wildwest.dev"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"cowboys"}, + }, + }}, + SideEffects: &sideEffect, + AdmissionReviewVersions: []string{"v1"}, + }}, } - responseAdmissionReview.Response = &t.Response - responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID - - respBytes, err := json.Marshal(responseAdmissionReview) - if err != nil { - t.T.Logf("%v", err) - http.Error(resp, err.Error(), http.StatusInternalServerError) - return + _, err = kubeClusterClient.Cluster(logicalClusters[0]).AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, webhook, metav1.CreateOptions{}) + require.NoError(t, err, "failed to add validating webhook configurations") + + cowboy := v1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing", + }, + Spec: v1alpha1.CowboySpec{}, } - t.Lock.Lock() - t.Calls = t.Calls + 1 - t.Lock.Unlock() + // Avoid race condition here by making sure that CRD is served after installing the types into logical clusters + t.Logf("Creating cowboy resource in second logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(logicalClusters[1]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + require.Equal(t, 0, testWebhook.Calls, "expected that the webhook is not called for logical cluster where webhook is not installed") - resp.Header().Set("Content-Type", "application/json") - if _, err := resp.Write(respBytes); err != nil { - t.T.Logf("%v", err) - } + t.Logf("Creating cowboy resource in first logical cluster") + require.Eventually(t, func() bool { + _, err = cowbyClients.Cluster(logicalClusters[0]).WildwestV1alpha1().Cowboys("default").Create(ctx, &cowboy, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return false + } + return true + + }, wait.ForeverTestTimeout, 100*time.Millisecond) + + //Avoid race condition where webhook informer is not updated before the call to create was made. + t.Log("Verify webhook is eventually called") + require.Eventually(t, func() bool { + return testWebhook.Calls == 1 + }, wait.ForeverTestTimeout, 100*time.Millisecond) } diff --git a/test/e2e/fixtures/webhook/webhook.go b/test/e2e/fixtures/webhook/webhook.go new file mode 100644 index 000000000000..e4f936fe133a --- /dev/null +++ b/test/e2e/fixtures/webhook/webhook.go @@ -0,0 +1,160 @@ +/* +Copyright 2022 Tkjjj KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package Webhook + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "path/filepath" + "sync" + "testing" + + "github.com/kcp-dev/kcp/test/e2e/framework" + v1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type WebhookServer struct { + Response v1.AdmissionResponse + ObjectGVK schema.GroupVersionKind + T *testing.T + Deserializer runtime.Decoder + + port string + + Lock sync.Mutex + Calls int +} + +func (t *WebhookServer) StartServer(ctx context.Context, server framework.RunningServer, port string) { + dirPath := filepath.Dir(server.KubeconfigPath()) + // using known path to cert and key + cfg := server.DefaultConfig(t.T) + cfg.CertFile = filepath.Join(dirPath, "apiserver.crt") + cfg.KeyFile = filepath.Join(dirPath, "apiserver.key") + t.port = port + serv := &http.Server{Addr: fmt.Sprintf(":%v", port), Handler: t} + go func() { + <-ctx.Done() + t.T.Logf("Shutting down the HTTP server") + err := serv.Shutdown(context.TODO()) + if err != nil { + t.T.Errorf("failure to shutdown server: %v", err) + } + }() + go func() { + err := serv.ListenAndServeTLS(cfg.CertFile, cfg.KeyFile) + if err != nil && err != http.ErrServerClosed { + t.T.Error(err) + } + }() +} + +func (t *WebhookServer) GetURL() *string { + s := fmt.Sprintf("https://localhost:%v/hello", t.port) + return &s +} + +func (t *WebhookServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + // Make sure that this is a request for the object that was set. + t.T.Log("made it webhook") + if req.Body == nil { + msg := "Expected request body to be non-empty" + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + } + + data, err := ioutil.ReadAll(req.Body) + if err != nil { + msg := fmt.Sprintf("Request could not be decoded: %v", err) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + } + + // verify the content type is accurate + contentType := req.Header.Get("Content-Type") + if contentType != "application/json" { + msg := fmt.Sprintf("contentType=%s, expect application/json", contentType) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + obj, gvk, err := t.Deserializer.Decode(data, nil, nil) + if err != nil { + msg := fmt.Sprintf("Unable to decode object: %v", err) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + if *gvk != v1.SchemeGroupVersion.WithKind("AdmissionReview") { + msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + requestedAdmissionReview, ok := obj.(*v1.AdmissionReview) + if !ok { + //return an error + msg := fmt.Sprintf("Expected AdmissionReview but got: %T", obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + obj, objGVK, err := t.Deserializer.Decode(requestedAdmissionReview.Request.Object.Raw, nil, nil) + if err != nil { + msg := fmt.Sprintf("Unable to decode admissions reqeusted object: %v", err) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + if t.ObjectGVK != *objGVK { + //return an error + msg := fmt.Sprintf("Expected ObjectGVK: %v but got: %T", t.ObjectGVK, obj) + t.T.Logf("%v", msg) + http.Error(resp, msg, http.StatusBadRequest) + return + } + + responseAdmissionReview := &v1.AdmissionReview{ + TypeMeta: requestedAdmissionReview.TypeMeta, + } + responseAdmissionReview.Response = &t.Response + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + + respBytes, err := json.Marshal(responseAdmissionReview) + if err != nil { + t.T.Logf("%v", err) + http.Error(resp, err.Error(), http.StatusInternalServerError) + return + } + + t.Lock.Lock() + t.Calls = t.Calls + 1 + t.Lock.Unlock() + + resp.Header().Set("Content-Type", "application/json") + if _, err := resp.Write(respBytes); err != nil { + t.T.Logf("%v", err) + } +}