From 0041a8318d1964863c0d6f5e56321fe5b47e7cfc Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Mon, 13 Jun 2022 15:19:13 +0200 Subject: [PATCH] test/extension: add first version of server lib & topology mutation extension --- .../api/v1alpha1/topologymutation_types.go | 2 +- .../patches/inline/json_patch_generator.go | 65 +--- .../inline/json_patch_generator_test.go | 27 -- .../cluster/patches/variables/merge.go | 75 +++++ .../cluster/patches/variables/merge_test.go | 51 +++ .../cluster/patches/variables/variables.go | 9 + test/e2e/cluster_upgrade_runtimesdk.go | 5 +- .../clusterclass-quick-start-runtimesdk.yaml | 4 +- .../handlers/topologymutation/handler.go | 88 ++++++ .../handlers/topologymutation/lib.go | 100 ++++++ test/extension/main.go | 58 +++- test/extension/server/server.go | 294 ++++++++++++++++++ test/go.mod | 2 +- 13 files changed, 671 insertions(+), 109 deletions(-) create mode 100644 internal/controllers/topology/cluster/patches/variables/merge.go create mode 100644 internal/controllers/topology/cluster/patches/variables/merge_test.go create mode 100644 test/extension/handlers/topologymutation/handler.go create mode 100644 test/extension/handlers/topologymutation/lib.go create mode 100644 test/extension/server/server.go diff --git a/exp/runtime/hooks/api/v1alpha1/topologymutation_types.go b/exp/runtime/hooks/api/v1alpha1/topologymutation_types.go index 13071cf188b0..ea90e6eab809 100644 --- a/exp/runtime/hooks/api/v1alpha1/topologymutation_types.go +++ b/exp/runtime/hooks/api/v1alpha1/topologymutation_types.go @@ -171,7 +171,7 @@ type HolderReference struct { } // ValidateTopology validates the Cluster topology after all patches have been applied. -func ValidateTopology(*GeneratePatchesRequest, *GeneratePatchesResponse) {} +func ValidateTopology(*ValidateTopologyRequest, *ValidateTopologyResponse) {} func init() { catalogBuilder.RegisterHook(GeneratePatches, &runtimecatalog.HookMeta{ diff --git a/internal/controllers/topology/cluster/patches/inline/json_patch_generator.go b/internal/controllers/topology/cluster/patches/inline/json_patch_generator.go index 628863be8bee..55dfc45687f9 100644 --- a/internal/controllers/topology/cluster/patches/inline/json_patch_generator.go +++ b/internal/controllers/topology/cluster/patches/inline/json_patch_generator.go @@ -54,14 +54,14 @@ func New(patch *clusterv1.ClusterClassPatch) api.Generator { func (j *jsonPatchGenerator) Generate(_ context.Context, req *runtimehooksv1.GeneratePatchesRequest) *runtimehooksv1.GeneratePatchesResponse { resp := &runtimehooksv1.GeneratePatchesResponse{} - globalVariables := toMap(req.Variables) + globalVariables := patchvariables.ToMap(req.Variables) // Loop over all templates. errs := []error{} for i := range req.Items { item := &req.Items[i] - templateVariables := toMap(item.Variables) + templateVariables := patchvariables.ToMap(item.Variables) // Calculate the list of patches which match the current template. matchingPatches := []clusterv1.PatchDefinition{} @@ -78,7 +78,7 @@ func (j *jsonPatchGenerator) Generate(_ context.Context, req *runtimehooksv1.Gen } // Merge template-specific and global variables. - variables, err := mergeVariableMaps(globalVariables, templateVariables) + variables, err := patchvariables.MergeVariableMaps(globalVariables, templateVariables) if err != nil { errs = append(errs, errors.Wrapf(err, "failed to merge global and template-specific variables for item with uid %q", item.UID)) continue @@ -124,15 +124,6 @@ func (j *jsonPatchGenerator) Generate(_ context.Context, req *runtimehooksv1.Gen return resp } -// toMap converts a list of Variables to a map of JSON (name is the map key). -func toMap(variables []runtimehooksv1.Variable) map[string]apiextensionsv1.JSON { - variablesMap := map[string]apiextensionsv1.JSON{} - for i := range variables { - variablesMap[variables[i].Name] = variables[i].Value - } - return variablesMap -} - // matchesSelector returns true if the GeneratePatchesRequestItem matches the selector. func matchesSelector(req *runtimehooksv1.GeneratePatchesRequestItem, templateVariables map[string]apiextensionsv1.JSON, selector clusterv1.PatchSelector) bool { gvk := req.Object.Object.GetObjectKind().GroupVersionKind() @@ -357,53 +348,3 @@ func calculateTemplateData(variables map[string]apiextensionsv1.JSON) (map[strin return res, nil } - -// mergeVariableMaps merges variables. -// NOTE: In case a variable exists in multiple maps, the variable from the latter map is preserved. -// NOTE: The builtin variable object is merged instead of simply overwritten. -func mergeVariableMaps(variableMaps ...map[string]apiextensionsv1.JSON) (map[string]apiextensionsv1.JSON, error) { - res := make(map[string]apiextensionsv1.JSON) - - for _, variableMap := range variableMaps { - for variableName, variableValue := range variableMap { - // If the variable already exits and is the builtin variable, merge it. - if _, ok := res[variableName]; ok && variableName == patchvariables.BuiltinsName { - mergedV, err := mergeBuiltinVariables(res[variableName], variableValue) - if err != nil { - return nil, errors.Wrapf(err, "failed to merge builtin variables") - } - res[variableName] = *mergedV - continue - } - res[variableName] = variableValue - } - } - - return res, nil -} - -// mergeBuiltinVariables merges builtin variable objects. -// NOTE: In case a variable exists in multiple builtin variables, the variable from the latter map is preserved. -func mergeBuiltinVariables(variableList ...apiextensionsv1.JSON) (*apiextensionsv1.JSON, error) { - builtins := &patchvariables.Builtins{} - - // Unmarshal all variables into builtins. - // NOTE: This accumulates the fields on the builtins. - // Fields will be overwritten by later Unmarshals if fields are - // set on multiple variables. - for _, variable := range variableList { - if err := json.Unmarshal(variable.Raw, builtins); err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal builtin variable") - } - } - - // Marshal builtins to JSON. - builtinVariableJSON, err := json.Marshal(builtins) - if err != nil { - return nil, errors.Wrapf(err, "failed to marshal builtin variable") - } - - return &apiextensionsv1.JSON{ - Raw: builtinVariableJSON, - }, nil -} diff --git a/internal/controllers/topology/cluster/patches/inline/json_patch_generator_test.go b/internal/controllers/topology/cluster/patches/inline/json_patch_generator_test.go index 3f31ee4856bc..b9e169f5436a 100644 --- a/internal/controllers/topology/cluster/patches/inline/json_patch_generator_test.go +++ b/internal/controllers/topology/cluster/patches/inline/json_patch_generator_test.go @@ -1759,33 +1759,6 @@ func TestCalculateTemplateData(t *testing.T) { } } -func TestMergeVariables(t *testing.T) { - t.Run("Merge variables", func(t *testing.T) { - g := NewWithT(t) - - m, err := mergeVariableMaps( - map[string]apiextensionsv1.JSON{ - patchvariables.BuiltinsName: {Raw: []byte(`{"cluster":{"name":"cluster-name","namespace":"default","topology":{"class":"clusterClass1","version":"v1.21.1"}}}`)}, - "a": {Raw: []byte("a-different")}, - "c": {Raw: []byte("c")}, - }, - map[string]apiextensionsv1.JSON{ - // Verify that builtin variables are merged correctly and - // the latter variables take precedent ("cluster-name-overwrite"). - patchvariables.BuiltinsName: {Raw: []byte(`{"controlPlane":{"replicas":3},"cluster":{"name":"cluster-name-overwrite"}}`)}, - "a": {Raw: []byte("a")}, - "b": {Raw: []byte("b")}, - }, - ) - g.Expect(err).To(BeNil()) - - g.Expect(m).To(HaveKeyWithValue(patchvariables.BuiltinsName, apiextensionsv1.JSON{Raw: []byte(`{"cluster":{"name":"cluster-name-overwrite","namespace":"default","topology":{"version":"v1.21.1","class":"clusterClass1"}},"controlPlane":{"replicas":3}}`)})) - g.Expect(m).To(HaveKeyWithValue("a", apiextensionsv1.JSON{Raw: []byte("a")})) - g.Expect(m).To(HaveKeyWithValue("b", apiextensionsv1.JSON{Raw: []byte("b")})) - g.Expect(m).To(HaveKeyWithValue("c", apiextensionsv1.JSON{Raw: []byte("c")})) - }) -} - // toJSONCompact is used to be able to write JSON values in a readable manner. func toJSONCompact(value string) []byte { var compactValue bytes.Buffer diff --git a/internal/controllers/topology/cluster/patches/variables/merge.go b/internal/controllers/topology/cluster/patches/variables/merge.go new file mode 100644 index 000000000000..3c1259635f82 --- /dev/null +++ b/internal/controllers/topology/cluster/patches/variables/merge.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package variables + +import ( + "encoding/json" + + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +// MergeVariableMaps merges variables. +// This func is useful when merging global and template-specific variables. +// NOTE: In case a variable exists in multiple maps, the variable from the latter map is preserved. +// NOTE: The builtin variable object is merged instead of simply overwritten. +func MergeVariableMaps(variableMaps ...map[string]apiextensionsv1.JSON) (map[string]apiextensionsv1.JSON, error) { + res := make(map[string]apiextensionsv1.JSON) + + for _, variableMap := range variableMaps { + for variableName, variableValue := range variableMap { + // If the variable already exits and is the builtin variable, merge it. + if _, ok := res[variableName]; ok && variableName == BuiltinsName { + mergedV, err := mergeBuiltinVariables(res[variableName], variableValue) + if err != nil { + return nil, errors.Wrapf(err, "failed to merge builtin variables") + } + res[variableName] = *mergedV + continue + } + res[variableName] = variableValue + } + } + + return res, nil +} + +// mergeBuiltinVariables merges builtin variable objects. +// NOTE: In case a variable exists in multiple builtin variables, the variable from the latter map is preserved. +func mergeBuiltinVariables(variableList ...apiextensionsv1.JSON) (*apiextensionsv1.JSON, error) { + builtins := &Builtins{} + + // Unmarshal all variables into builtins. + // NOTE: This accumulates the fields on the builtins. + // Fields will be overwritten by later Unmarshals if fields are + // set on multiple variables. + for _, variable := range variableList { + if err := json.Unmarshal(variable.Raw, builtins); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal builtin variable") + } + } + + // Marshal builtins to JSON. + builtinVariableJSON, err := json.Marshal(builtins) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal builtin variable") + } + + return &apiextensionsv1.JSON{ + Raw: builtinVariableJSON, + }, nil +} diff --git a/internal/controllers/topology/cluster/patches/variables/merge_test.go b/internal/controllers/topology/cluster/patches/variables/merge_test.go new file mode 100644 index 000000000000..dc2a4b5793a9 --- /dev/null +++ b/internal/controllers/topology/cluster/patches/variables/merge_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package variables + +import ( + "testing" + + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func TestMergeVariables(t *testing.T) { + t.Run("Merge variables", func(t *testing.T) { + g := NewWithT(t) + + m, err := MergeVariableMaps( + map[string]apiextensionsv1.JSON{ + BuiltinsName: {Raw: []byte(`{"cluster":{"name":"cluster-name","namespace":"default","topology":{"class":"clusterClass1","version":"v1.21.1"}}}`)}, + "a": {Raw: []byte("a-different")}, + "c": {Raw: []byte("c")}, + }, + map[string]apiextensionsv1.JSON{ + // Verify that builtin variables are merged correctly and + // the latter variables take precedent ("cluster-name-overwrite"). + BuiltinsName: {Raw: []byte(`{"controlPlane":{"replicas":3},"cluster":{"name":"cluster-name-overwrite"}}`)}, + "a": {Raw: []byte("a")}, + "b": {Raw: []byte("b")}, + }, + ) + g.Expect(err).To(BeNil()) + + g.Expect(m).To(HaveKeyWithValue(BuiltinsName, apiextensionsv1.JSON{Raw: []byte(`{"cluster":{"name":"cluster-name-overwrite","namespace":"default","topology":{"version":"v1.21.1","class":"clusterClass1"}},"controlPlane":{"replicas":3}}`)})) + g.Expect(m).To(HaveKeyWithValue("a", apiextensionsv1.JSON{Raw: []byte("a")})) + g.Expect(m).To(HaveKeyWithValue("b", apiextensionsv1.JSON{Raw: []byte("b")})) + g.Expect(m).To(HaveKeyWithValue("c", apiextensionsv1.JSON{Raw: []byte("c")})) + }) +} diff --git a/internal/controllers/topology/cluster/patches/variables/variables.go b/internal/controllers/topology/cluster/patches/variables/variables.go index 4090bc88dae4..fe618e1475f3 100644 --- a/internal/controllers/topology/cluster/patches/variables/variables.go +++ b/internal/controllers/topology/cluster/patches/variables/variables.go @@ -346,3 +346,12 @@ func ipFamilyToString(ipFamily clusterv1.ClusterIPFamily) string { return "Invalid" } } + +// ToMap converts a list of Variables to a map of JSON (name is the map key). +func ToMap(variables []runtimehooksv1.Variable) map[string]apiextensionsv1.JSON { + variablesMap := map[string]apiextensionsv1.JSON{} + for i := range variables { + variablesMap[variables[i].Name] = variables[i].Value + } + return variablesMap +} diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go index 98a02a11425d..3444644e0ddc 100644 --- a/test/e2e/cluster_upgrade_runtimesdk.go +++ b/test/e2e/cluster_upgrade_runtimesdk.go @@ -214,7 +214,10 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl func extensionConfig(specName string, namespace *corev1.Namespace) *runtimev1.ExtensionConfig { return &runtimev1.ExtensionConfig{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + // FIXME(sbueringer): use constant name for now as we have to be able to reference it in the ClusterClass. + // Random generate later on again when Yuvaraj's PR has split up the cluster lifecycle. + //Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + Name: specName, Annotations: map[string]string{ runtimev1.InjectCAFromSecretAnnotation: fmt.Sprintf("%s/webhook-service-cert", namespace.Name), }, diff --git a/test/e2e/data/infrastructure-docker/v1beta1/clusterclass-quick-start-runtimesdk.yaml b/test/e2e/data/infrastructure-docker/v1beta1/clusterclass-quick-start-runtimesdk.yaml index f110589eb84f..054631192f74 100644 --- a/test/e2e/data/infrastructure-docker/v1beta1/clusterclass-quick-start-runtimesdk.yaml +++ b/test/e2e/data/infrastructure-docker/v1beta1/clusterclass-quick-start-runtimesdk.yaml @@ -75,8 +75,8 @@ spec: # TODO: enable external patches once topology mutation is implemented # - name: lbImageRepository # external: -# generateExtension: generate-patches.test-extension-config -# validateExtension: validate-topology.test-extension-config +# generateExtension: generate-patches.k8s-upgrade-with-runtimesdk +# validateExtension: validate-topology.k8s-upgrade-with-runtimesdk - name: imageRepository description: "Sets the imageRepository used for the KubeadmControlPlane." definitions: diff --git a/test/extension/handlers/topologymutation/handler.go b/test/extension/handlers/topologymutation/handler.go new file mode 100644 index 000000000000..805adc47e5c4 --- /dev/null +++ b/test/extension/handlers/topologymutation/handler.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package topologymutation contains the handlers for the topologymutation webhook. +package topologymutation + +import ( + "context" + "strconv" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + ctrl "sigs.k8s.io/controller-runtime" + + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + patchvariables "sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/patches/variables" + infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" +) + +// NewHandler returns a new topology mutation Handler. +func NewHandler(scheme *runtime.Scheme) *Handler { + return &Handler{ + decoder: serializer.NewCodecFactory(scheme).UniversalDecoder( + infrav1.GroupVersion, + ), + } +} + +// Handler is a topology mutation handler. +type Handler struct { + decoder runtime.Decoder +} + +// GeneratePatches returns a function that generates patches for the given request. +func (h *Handler) GeneratePatches(ctx context.Context, req *runtimehooksv1.GeneratePatchesRequest, resp *runtimehooksv1.GeneratePatchesResponse) { + log := ctrl.LoggerFrom(ctx) + log.Info("GeneratePatches called") + + walkTemplates(h.decoder, req, resp, func(obj runtime.Object, variables map[string]apiextensionsv1.JSON) error { + if dockerClusterTemplate, ok := obj.(*infrav1.DockerClusterTemplate); ok { + if err := patchDockerClusterTemplate(dockerClusterTemplate, variables); err != nil { + return err + } + } + + return nil + }) +} + +// patchDockerClusterTemplate patches the DockerClusterTepmlate. +func patchDockerClusterTemplate(dockerClusterTemplate *infrav1.DockerClusterTemplate, templateVariables map[string]apiextensionsv1.JSON) error { + // Get the variable value as JSON string. + value, err := patchvariables.GetVariableValue(templateVariables, "lbImageRepository") + if err != nil { + return err + } + + // Unquote the JSON string. + stringValue, err := strconv.Unquote(string(value.Raw)) + if err != nil { + return err + } + + dockerClusterTemplate.Spec.Template.Spec.LoadBalancer.ImageRepository = stringValue + return nil +} + +// ValidateTopology returns a function that validates the given request. +func (h *Handler) ValidateTopology(ctx context.Context, _ *runtimehooksv1.ValidateTopologyRequest, resp *runtimehooksv1.ValidateTopologyResponse) { + log := ctrl.LoggerFrom(ctx) + log.Info("ValidateTopology called") + + resp.Status = runtimehooksv1.ResponseStatusSuccess +} diff --git a/test/extension/handlers/topologymutation/lib.go b/test/extension/handlers/topologymutation/lib.go new file mode 100644 index 000000000000..66f7eac31305 --- /dev/null +++ b/test/extension/handlers/topologymutation/lib.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topologymutation + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + "gomodules.xyz/jsonpatch/v2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + patchvariables "sigs.k8s.io/cluster-api/internal/controllers/topology/cluster/patches/variables" +) + +// walkTemplates walks through all templates of a GeneratePatchesRequest and calls the mutateFunc. +func walkTemplates(decoder runtime.Decoder, req *runtimehooksv1.GeneratePatchesRequest, resp *runtimehooksv1.GeneratePatchesResponse, mutateFunc func(obj runtime.Object, variables map[string]apiextensionsv1.JSON) error) { + globalVariables := patchvariables.ToMap(req.Variables) + + for _, requestItem := range req.Items { + templateVariables, err := patchvariables.MergeVariableMaps(globalVariables, patchvariables.ToMap(requestItem.Variables)) + if err != nil { + resp.Status = runtimehooksv1.ResponseStatusFailure + resp.Message = err.Error() + return + } + + obj, _, err := decoder.Decode(requestItem.Object.Raw, nil, requestItem.Object.Object) + if err != nil { + // Continue, object has a type which hasn't been registered with the scheme. + continue + } + + original := obj.DeepCopyObject() + + if err := mutateFunc(obj, templateVariables); err != nil { + resp.Status = runtimehooksv1.ResponseStatusFailure + resp.Message = err.Error() + return + } + + patch, err := createPatch(original, obj) + if err != nil { + resp.Status = runtimehooksv1.ResponseStatusFailure + resp.Message = err.Error() + return + } + + resp.Items = append(resp.Items, runtimehooksv1.GeneratePatchesResponseItem{ + UID: requestItem.UID, + PatchType: runtimehooksv1.JSONPatchType, + Patch: patch, + }) + + fmt.Printf("Generated patch (uid: %q): %q\n", requestItem.UID, string(patch)) + } + + resp.Status = runtimehooksv1.ResponseStatusSuccess +} + +// createPatch creates a JSON patch from the original and the modified object. +func createPatch(original, modified runtime.Object) ([]byte, error) { + marshalledOriginal, err := json.Marshal(original) + if err != nil { + return nil, errors.Errorf("failed to marshal original object: %v", err) + } + + marshalledModified, err := json.Marshal(modified) + if err != nil { + return nil, errors.Errorf("failed to marshal modified object: %v", err) + } + + patch, err := jsonpatch.CreatePatch(marshalledOriginal, marshalledModified) + if err != nil { + return nil, errors.Errorf("failed to create patch: %v", err) + } + + patchBytes, err := json.Marshal(patch) + if err != nil { + return nil, errors.Errorf("failed to marshal patch: %v", err) + } + + return patchBytes, nil +} diff --git a/test/extension/main.go b/test/extension/main.go index 3e57c48f5b14..3aa9571d829b 100644 --- a/test/extension/main.go +++ b/test/extension/main.go @@ -26,18 +26,21 @@ import ( cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/logs" "k8s.io/klog/v2" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog" + "sigs.k8s.io/cluster-api/test/extension/handlers/topologymutation" + "sigs.k8s.io/cluster-api/test/extension/server" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" "sigs.k8s.io/cluster-api/version" ) var ( - catalog = runtimecatalog.New() - scheme = runtime.NewScheme() + catalog = runtimecatalog.New() + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") // flags. @@ -92,22 +95,47 @@ func main() { ctx := ctrl.SetupSignalHandler() - srv := webhook.Server{ - Host: "", - Port: webhookPort, - CertDir: webhookCertDir, - CertName: "tls.crt", - KeyName: "tls.key", - WebhookMux: http.NewServeMux(), - TLSMinVersion: "1.2", + webhookServer, err := server.NewServer(server.Options{ + Catalog: catalog, + Port: webhookPort, + CertDir: webhookCertDir, + }) + if err != nil { + setupLog.Error(err, "error creating webhook server") + os.Exit(1) + } + + topologyMutationHandler := topologymutation.NewHandler(scheme) + + if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + Hook: runtimehooksv1.GeneratePatches, + Name: "generate-patches", + HandlerFunc: topologyMutationHandler.GeneratePatches, + TimeoutSeconds: pointer.Int32(5), + FailurePolicy: toPtr(runtimehooksv1.FailurePolicyFail), + }); err != nil { + setupLog.Error(err, "error adding handler") + os.Exit(1) } - // TODO: next PRs - // srv.WebhookMux.Handle("/", operation1Handler) + if err := webhookServer.AddExtensionHandler(server.ExtensionHandler{ + Hook: runtimehooksv1.ValidateTopology, + Name: "validate-topology", + HandlerFunc: topologyMutationHandler.ValidateTopology, + TimeoutSeconds: pointer.Int32(5), + FailurePolicy: toPtr(runtimehooksv1.FailurePolicyFail), + }); err != nil { + setupLog.Error(err, "error adding handler") + os.Exit(1) + } setupLog.Info("starting RuntimeExtension", "version", version.Get().String()) - if err := srv.StartStandalone(ctx, nil); err != nil { - setupLog.Error(err, "problem running webhook") + if err := webhookServer.Start(ctx); err != nil { + setupLog.Error(err, "error running webhook server") os.Exit(1) } } + +func toPtr(f runtimehooksv1.FailurePolicy) *runtimehooksv1.FailurePolicy { + return &f +} diff --git a/test/extension/server/server.go b/test/extension/server/server.go new file mode 100644 index 000000000000..7e0f0d4791b7 --- /dev/null +++ b/test/extension/server/server.go @@ -0,0 +1,294 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package server contains the implementation of a RuntimeSDK webhook server. +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "reflect" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + runtimecatalog "sigs.k8s.io/cluster-api/internal/runtime/catalog" +) + +const tlsVersion13 = "1.3" + +// DefaultPort is the default port that the webhook server serves. +var DefaultPort = 9443 + +// Server is a runtime webhook server. +type Server struct { + catalog *runtimecatalog.Catalog + server *webhook.Server + handlers map[string]ExtensionHandler +} + +// Options are the options for the Server. +type Options struct { + // Catalog is the catalog used to handle requests. + Catalog *runtimecatalog.Catalog + + // Port is the port that the webhook server serves at. + // It is used to set webhook.Server.Port. + Port int + + // Host is the hostname that the webhook server binds to. + // It is used to set webhook.Server.Host. + Host string + + // CertDir is the directory that contains the server key and certificate. + // If not set, webhook server would look up the server key and certificate in + // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate + // must be named tls.key and tls.crt, respectively. + // It is used to set webhook.Server.CertDir. + CertDir string +} + +// NewServer creates a new runtime webhook server based on the given Options. +func NewServer(options Options) (*Server, error) { + if options.Catalog == nil { + return nil, errors.Errorf("catalog is required") + } + if options.Port <= 0 { + options.Port = DefaultPort + } + if options.CertDir == "" { + options.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs") + } + + webhookServer := &webhook.Server{ + Port: options.Port, + Host: options.Host, + CertDir: options.CertDir, + CertName: "tls.crt", + KeyName: "tls.key", + WebhookMux: http.NewServeMux(), + TLSMinVersion: tlsVersion13, + } + + return &Server{ + catalog: options.Catalog, + server: webhookServer, + handlers: map[string]ExtensionHandler{}, + }, nil +} + +// ExtensionHandler represents an extension handler. +type ExtensionHandler struct { + // gvh is the gvh of the hook corresponding to the extension handler. + gvh runtimecatalog.GroupVersionHook + // requestObject is a runtime object that the handler expects to receive. + requestObject runtime.Object + // responseObject is a runtime object that the handler expects to return. + responseObject runtime.Object + + // Hook is the corresponding hook of the handler. + Hook runtimecatalog.Hook + + // Name is the name of the extension handler. + Name string + + // HandlerFunc is the handler function. + HandlerFunc runtimecatalog.Hook + + // TimeoutSeconds is the timeout of the extension handler. + TimeoutSeconds *int32 + + // FailurePolicy is the failure policy of the extension handler + FailurePolicy *runtimehooksv1.FailurePolicy +} + +// AddExtensionHandler adds an extension handler to the server. +func (s *Server) AddExtensionHandler(handler ExtensionHandler) error { + gvh, err := s.catalog.GroupVersionHook(handler.Hook) + if err != nil { + return errors.Wrapf(err, "hook %q does not exist in catalog", runtimecatalog.HookName(handler.Hook)) + } + handler.gvh = gvh + + requestObject, err := s.catalog.NewRequest(handler.gvh) + if err != nil { + return err + } + handler.requestObject = requestObject + + responseObject, err := s.catalog.NewResponse(handler.gvh) + if err != nil { + return err + } + handler.responseObject = responseObject + + if err := s.validateHandler(handler); err != nil { + return err + } + + handlerPath := runtimecatalog.GVHToPath(handler.gvh, handler.Name) + if _, ok := s.handlers[handlerPath]; ok { + return errors.Errorf("there is already a handler registered for path %q", handlerPath) + } + + s.handlers[handlerPath] = handler + return nil +} + +// validateHandler validates a handler. +func (s *Server) validateHandler(handler ExtensionHandler) error { + // Get hook and handler type. + hookFuncType := reflect.TypeOf(handler.Hook) + handlerFuncType := reflect.TypeOf(handler.HandlerFunc) + + // Validate handler function signature. + if handlerFuncType.Kind() != reflect.Func { + return errors.Errorf("HandlerFunc must be a func") + } + if handlerFuncType.NumIn() != 3 { + return errors.Errorf("HandlerFunc must have three input parameter") + } + if handlerFuncType.NumOut() != 0 { + return errors.Errorf("HandlerFunc must have no output parameter") + } + + // Get hook and handler request and response types. + hookRequestType := hookFuncType.In(0) //nolint:ifshort + hookResponseType := hookFuncType.In(1) //nolint:ifshort + handlerContextType := handlerFuncType.In(0) + handlerRequestType := handlerFuncType.In(1) + handlerResponseType := handlerFuncType.In(2) + + // Validate handler request and response are pointers. + if handlerRequestType.Kind() != reflect.Ptr { + return errors.Errorf("HandlerFunc request type must be a pointer") + } + if handlerResponseType.Kind() != reflect.Ptr { + return errors.Errorf("HandlerFunc response type must be a pointer") + } + + // Validate first handler parameter is a context + // TODO: improve check, how to check if param is a specific interface? + if handlerContextType.Name() != "Context" { + return errors.Errorf("HandlerFunc first parameter must be Context but is %s", handlerContextType.Name()) + } + + // Validate hook and handler request and response types are equal. + if hookRequestType != handlerRequestType { + return errors.Errorf("HandlerFunc request type must be *%s but is *%s", hookRequestType.Elem().Name(), handlerRequestType.Elem().Name()) + } + if hookResponseType != handlerResponseType { + return errors.Errorf("HandlerFunc response type must be *%s but is *%s", hookResponseType.Elem().Name(), handlerResponseType.Elem().Name()) + } + + return nil +} + +// Start starts the server. +func (s *Server) Start(ctx context.Context) error { + // Add discovery handler. + err := s.AddExtensionHandler(ExtensionHandler{ + Hook: runtimehooksv1.Discovery, + HandlerFunc: discoveryHandler(s.handlers), + }) + if err != nil { + return err + } + + // Add handlers to router. + for handlerPath, h := range s.handlers { + handler := h + + wrappedHandler := s.wrapHandler(handler) + s.server.Register(handlerPath, http.HandlerFunc(wrappedHandler)) + } + + return s.server.StartStandalone(ctx, nil) +} + +// discoveryHandler generates a discovery handler based on a list of handlers. +func discoveryHandler(handlers map[string]ExtensionHandler) func(context.Context, *runtimehooksv1.DiscoveryRequest, *runtimehooksv1.DiscoveryResponse) { + return func(_ context.Context, request *runtimehooksv1.DiscoveryRequest, response *runtimehooksv1.DiscoveryResponse) { + response.Status = runtimehooksv1.ResponseStatusSuccess + + for _, handler := range handlers { + response.Handlers = append(response.Handlers, runtimehooksv1.ExtensionHandler{ + Name: handler.Name, + RequestHook: runtimehooksv1.GroupVersionHook{ + APIVersion: handler.gvh.GroupVersion().String(), + Hook: handler.gvh.Hook, + }, + TimeoutSeconds: handler.TimeoutSeconds, + FailurePolicy: handler.FailurePolicy, + }) + } + } +} + +func (s *Server) wrapHandler(handler ExtensionHandler) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + response := s.callHandler(handler, r) + + responseBody, err := json.Marshal(response) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprintf(w, "unable to marshal response: %v", err) + return + } + + w.WriteHeader(http.StatusOK) + _, _ = w.Write(responseBody) + } +} + +func (s *Server) callHandler(handler ExtensionHandler, r *http.Request) runtimehooksv1.ResponseObject { + request := handler.requestObject.DeepCopyObject() + response := handler.responseObject.DeepCopyObject().(runtimehooksv1.ResponseObject) + + requestBody, err := io.ReadAll(r.Body) + if err != nil { + response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(fmt.Sprintf("error reading request: %v", err)) + return response + } + + if err := json.Unmarshal(requestBody, request); err != nil { + response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(fmt.Sprintf("error unmarshalling request: %v", err)) + return response + } + + // log.Log is the logger previously set via ctrl.SetLogger. + // This implemented analog to the logger in the controller-runtime manager. + ctx := ctrl.LoggerInto(r.Context(), log.Log) + + reflect.ValueOf(handler.HandlerFunc).Call([]reflect.Value{ + reflect.ValueOf(ctx), + reflect.ValueOf(request), + reflect.ValueOf(response), + }) + + return response +} diff --git a/test/go.mod b/test/go.mod index 52673760f28a..543e41bd0261 100644 --- a/test/go.mod +++ b/test/go.mod @@ -15,6 +15,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 github.com/vincent-petithory/dataurl v1.0.0 + gomodules.xyz/jsonpatch/v2 v2.2.0 k8s.io/api v0.24.0 k8s.io/apiextensions-apiserver v0.24.0 k8s.io/apimachinery v0.24.0 @@ -115,7 +116,6 @@ require ( golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect google.golang.org/protobuf v1.27.1 // indirect