From 1c5289949925d375cd8bcc88ab1f65b5bab8ad53 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Fri, 29 Oct 2021 15:56:55 +0200 Subject: [PATCH] Add ClusterClass patch engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Büringer buringerst@vmware.com Co-authored-by: fabriziopandini --- controllers/topology/cluster_controller.go | 6 + controllers/topology/desired_state.go | 31 +- .../extensions/patches/api/interface.go | 138 ++++ .../internal/extensions/patches/engine.go | 390 ++++++++++++ .../extensions/patches/engine_test.go | 595 ++++++++++++++++++ .../internal/extensions/patches/patch.go | 211 +++++++ .../internal/extensions/patches/patch_test.go | 369 +++++++++++ .../internal/extensions/patches/template.go | 165 +++++ .../extensions/patches/variables/variables.go | 192 ++++++ .../patches/variables/variables_test.go | 190 ++++++ controllers/topology/internal/log/log.go | 9 + ...56-cluster-class-and-managed-topologies.md | 8 +- internal/builder/builders.go | 27 +- 13 files changed, 2314 insertions(+), 17 deletions(-) create mode 100644 controllers/topology/internal/extensions/patches/api/interface.go create mode 100644 controllers/topology/internal/extensions/patches/engine.go create mode 100644 controllers/topology/internal/extensions/patches/engine_test.go create mode 100644 controllers/topology/internal/extensions/patches/patch.go create mode 100644 controllers/topology/internal/extensions/patches/patch_test.go create mode 100644 controllers/topology/internal/extensions/patches/template.go create mode 100644 controllers/topology/internal/extensions/patches/variables/variables.go create mode 100644 controllers/topology/internal/extensions/patches/variables/variables_test.go diff --git a/controllers/topology/cluster_controller.go b/controllers/topology/cluster_controller.go index 66e7e2748e8c..e011b9062d2c 100644 --- a/controllers/topology/cluster_controller.go +++ b/controllers/topology/cluster_controller.go @@ -26,6 +26,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/api/v1beta1/index" "sigs.k8s.io/cluster-api/controllers/external" + "sigs.k8s.io/cluster-api/controllers/topology/internal/extensions/patches" "sigs.k8s.io/cluster-api/controllers/topology/internal/scope" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" @@ -58,6 +59,9 @@ type ClusterReconciler struct { UnstructuredCachingClient client.Client externalTracker external.ObjectTracker + + // patchEngine is used to apply patches during computeDesiredState. + patchEngine patches.Engine } func (r *ClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { @@ -83,6 +87,8 @@ func (r *ClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manag r.externalTracker = external.ObjectTracker{ Controller: c, } + r.patchEngine = patches.NewEngine() + return nil } diff --git a/controllers/topology/desired_state.go b/controllers/topology/desired_state.go index ceaeb072aef3..80675e201d74 100644 --- a/controllers/topology/desired_state.go +++ b/controllers/topology/desired_state.go @@ -36,7 +36,7 @@ import ( // computeDesiredState computes the desired state of the cluster topology. // NOTE: We are assuming all the required objects are provided as input; also, in case of any error, -// the entire compute operation operation will fail. This might be improved in the future if support for reconciling +// the entire compute operation will fail. This might be improved in the future if support for reconciling // subset of a topology will be implemented. func (r *ClusterReconciler) computeDesiredState(ctx context.Context, s *scope.Scope) (*scope.ClusterState, error) { var err error @@ -46,36 +46,43 @@ func (r *ClusterReconciler) computeDesiredState(ctx context.Context, s *scope.Sc // Compute the desired state of the InfrastructureCluster object. if desiredState.InfrastructureCluster, err = computeInfrastructureCluster(ctx, s); err != nil { - return nil, err + return nil, errors.Wrapf(err, "failed to compute InfrastructureCluster") } // If the clusterClass mandates the controlPlane has infrastructureMachines, compute the InfrastructureMachineTemplate for the ControlPlane. if s.Blueprint.HasControlPlaneInfrastructureMachine() { if desiredState.ControlPlane.InfrastructureMachineTemplate, err = computeControlPlaneInfrastructureMachineTemplate(ctx, s); err != nil { - return nil, err + return nil, errors.Wrapf(err, "failed to compute ControlPlane InfrastructureMachineTemplate") } } // Compute the desired state of the ControlPlane object, eventually adding a reference to the // InfrastructureMachineTemplate generated by the previous step. if desiredState.ControlPlane.Object, err = computeControlPlane(ctx, s, desiredState.ControlPlane.InfrastructureMachineTemplate); err != nil { - return nil, err + return nil, errors.Wrapf(err, "failed to compute ControlPlane") } // Compute the desired state for the Cluster object adding a reference to the // InfrastructureCluster and the ControlPlane objects generated by the previous step. desiredState.Cluster = computeCluster(ctx, s, desiredState.InfrastructureCluster, desiredState.ControlPlane.Object) - // If required by the blueprint, compute the desired state of the MachineDeployment objects for the worker nodes, if any. - if !s.Blueprint.HasMachineDeployments() { - return desiredState, nil + // If required, compute the desired state of the MachineDeployments from the list of MachineDeploymentTopologies + // defined in the cluster. + if s.Blueprint.HasMachineDeployments() { + desiredState.MachineDeployments, err = computeMachineDeployments(ctx, s, desiredState.ControlPlane) + if err != nil { + return nil, errors.Wrapf(err, "failed to compute MachineDeployments") + } } - // Compute the desired state of the MachineDeployments from the list of MachineDeploymentTopologies - // defined in the cluster. - desiredState.MachineDeployments, err = computeMachineDeployments(ctx, s, desiredState.ControlPlane) - if err != nil { - return nil, err + // Apply patches the desired state according to the patches from the ClusterClass, variables from the Cluster + // and builtin variables. + // NOTE: We have to make sure all spec fields that were explicitly set in desired objects during the computation above + // are preserved during patching. When desired objects are computed their spec is copied from a template, in some cases + // further modifications to the spec are made afterwards. In those cases we have to make sure those fields are not overwritten + // in apply patches. Some examples are .spec.machineTemplate and .spec.version in control planes. + if err := r.patchEngine.Apply(ctx, s.Blueprint, desiredState); err != nil { + return nil, errors.Wrap(err, "failed to apply patches") } return desiredState, nil diff --git a/controllers/topology/internal/extensions/patches/api/interface.go b/controllers/topology/internal/extensions/patches/api/interface.go new file mode 100644 index 000000000000..4130c445e7fb --- /dev/null +++ b/controllers/topology/internal/extensions/patches/api/interface.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package api contains the API definition for the patch engine. +// NOTE: We are introducing this API as a decoupling layer between the patch engine and the concrete components +// responsible for generating patches, because we aim to provide support for external patches in a future iteration. +// We also assume that this API and all the related types will be moved in a separate (versioned) package thus +// providing a versioned contract between Cluster API and the components implementing external patch extensions. +package api + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +// Generator defines a component that can generate patches for ClusterClass templates. +type Generator interface { + // Generate generates patches for templates. + // GenerateRequest contains templates and the corresponding variables. + // GenerateResponse contains the patches which should be applied to the templates of the GenerateRequest. + Generate(ctx context.Context, request *GenerateRequest) (*GenerateResponse, error) +} + +// GenerateRequest defines the input for a Generate request. +type GenerateRequest struct { + // Variables is a name/value map containing variables. + Variables map[string]apiextensionsv1.JSON + + // Items contains the list of templates to generate patches for. + Items []*GenerateRequestTemplate +} + +// GenerateRequestTemplate defines one of the ClusterClass templates to generate patches for. +type GenerateRequestTemplate struct { + // TemplateRef identifies a template to generate patches for; + // the same TemplateRef must be used when specifying to which template a generated patch should be applied to. + TemplateRef TemplateRef + + // Variables is a name/value map containing variables specifically for the current template. + // For example some builtin variables like MachineDeployment replicas and version are context-sensitive + // and thus are only added to templates for MachineDeployments and with values which correspond to the + // current MachineDeployment. + Variables map[string]apiextensionsv1.JSON + + // Template contains the template. + Template apiextensionsv1.JSON +} + +// TemplateRef identifies one of the ClusterClass templates to generate patches for; +// the same TemplateRef must be used when specifying where a generated patch should apply to. +type TemplateRef struct { + // APIVersion of the current template. + APIVersion string + + // Kind of the current template. + Kind string + + // TemplateType defines where the template is used. + TemplateType TemplateType + + // MachineDeployment specifies the MachineDeployment in which the template is used. + // This field is only set if the template is used in the context of a MachineDeployment. + MachineDeploymentRef MachineDeploymentRef +} + +// MachineDeploymentRef specifies the MachineDeployment in which the template is used. +type MachineDeploymentRef struct { + // TopologyName is the name of the MachineDeploymentTopology. + TopologyName string + + // Class is the name of the MachineDeploymentClass. + Class string +} + +// TemplateType define the type for target types enum. +type TemplateType string + +const ( + // InfrastructureClusterTemplateType identifies a template for the InfrastructureCluster object. + InfrastructureClusterTemplateType TemplateType = "InfrastructureClusterTemplate" + + // ControlPlaneTemplateType identifies a template for the ControlPlane object. + ControlPlaneTemplateType TemplateType = "ControlPlaneTemplate" + + // ControlPlaneInfrastructureMachineTemplateType identifies a template for the InfrastructureMachines to be used for the ControlPlane object. + ControlPlaneInfrastructureMachineTemplateType TemplateType = "ControlPlane/InfrastructureMachineTemplate" + + // MachineDeploymentBootstrapConfigTemplateType identifies a template for the BootstrapConfig to be used for a MachineDeployment object. + MachineDeploymentBootstrapConfigTemplateType TemplateType = "MachineDeployment/BootstrapConfigTemplate" + + // MachineDeploymentInfrastructureMachineTemplateType identifies a template for the InfrastructureMachines to be used for a MachineDeployment object. + MachineDeploymentInfrastructureMachineTemplateType TemplateType = "MachineDeployment/InfrastructureMachineTemplate" +) + +// PatchType define the type for patch types enum. +type PatchType string + +const ( + // JSONPatchType identifies a https://datatracker.ietf.org/doc/html/rfc6902 json patch. + JSONPatchType PatchType = "JSONPatch" + + // JSONMergePatchType identifies a https://datatracker.ietf.org/doc/html/rfc7386 json merge patch. + JSONMergePatchType PatchType = "JSONMergePatch" +) + +// GenerateResponse defines the response of a Generate request. +// NOTE: Patches defined in GenerateResponse will be applied in the same order to the original +// GenerateRequest object, thus adding changes on templates across all the subsequent Generate calls. +type GenerateResponse struct { + // Items contains the list of generated patches. + Items []GenerateResponsePatch +} + +// GenerateResponsePatch defines a Patch targeting a specific GenerateRequestTemplate. +type GenerateResponsePatch struct { + // TemplateRef identifies the template the patch should apply to. + TemplateRef TemplateRef + + // Patch contains the patch. + Patch apiextensionsv1.JSON + + // Patch defines the type of the JSON patch. + PatchType PatchType +} diff --git a/controllers/topology/internal/extensions/patches/engine.go b/controllers/topology/internal/extensions/patches/engine.go new file mode 100644 index 000000000000..9b5bd3b2352e --- /dev/null +++ b/controllers/topology/internal/extensions/patches/engine.go @@ -0,0 +1,390 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package patches implement the patch engine. +package patches + +import ( + "context" + + jsonpatch "github.com/evanphx/json-patch/v5" + "github.com/pkg/errors" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/topology/internal/contract" + "sigs.k8s.io/cluster-api/controllers/topology/internal/extensions/patches/api" + "sigs.k8s.io/cluster-api/controllers/topology/internal/extensions/patches/variables" + tlog "sigs.k8s.io/cluster-api/controllers/topology/internal/log" + "sigs.k8s.io/cluster-api/controllers/topology/internal/scope" +) + +// Engine is a patch engine which applies patches defined in a ClusterBlueprint to a ClusterState. +type Engine interface { + Apply(ctx context.Context, blueprint *scope.ClusterBlueprint, desired *scope.ClusterState) error +} + +// NewEngine creates a new patch engine. +func NewEngine() Engine { + return &engine{ + createPatchGenerator: createPatchGenerator, + } +} + +// engine implements the Engine interface. +type engine struct { + // createPatchGenerator is the func which returns a patch generator + // based on a ClusterClassPatch. + // Note: This field is also used to inject patches in unit tests. + createPatchGenerator func(patch *clusterv1.ClusterClassPatch) (api.Generator, error) +} + +// Apply applies patches to the desired state according to the patches from the ClusterClass, variables from the Cluster +// and builtin variables. +// * A GenerateRequest with all templates and global and template-specific variables is created. +// * Then for all ClusterClassPatches of a ClusterClass, JSON or JSON merge patches are generated +// and successively applied to the templates in the GenerateRequest. +// * Eventually the patched templates are used to update the specs of the desired objects. +func (e *engine) Apply(ctx context.Context, blueprint *scope.ClusterBlueprint, desired *scope.ClusterState) error { + // Return if there are no patches. + if len(blueprint.ClusterClass.Spec.Patches) == 0 { + return nil + } + + log := tlog.LoggerFrom(ctx) + + // Create a patch generation request. + req, err := createRequest(blueprint, desired) + if err != nil { + return errors.Wrapf(err, "failed to generate patch request") + } + + // Loop over patches in ClusterClass, generate patches and apply them to the request, + // respecting the order in which they are defined. + for i := range blueprint.ClusterClass.Spec.Patches { + clusterClassPatch := blueprint.ClusterClass.Spec.Patches[i] + ctx, log = log.WithValues("patch", clusterClassPatch.Name).Into(ctx) + + log.V(5).Infof("Applying patch to templates") + + // Create patch generator for the current patch. + generator, err := e.createPatchGenerator(&clusterClassPatch) + if err != nil { + return err + } + + // Generate patches. + // NOTE: All the partial patches accumulate on top of the request, so the + // patch generator in the next iteration of the loop will get the modified + // version of the request (including the patched version of the templates). + resp, err := generator.Generate(ctx, req) + if err != nil { + return errors.Errorf("failed to generate patches for patch %q", clusterClassPatch.Name) + } + + // Apply patches to the request. + if err := applyPatchesToRequest(ctx, req, resp); err != nil { + return err + } + } + + // Use patched templates to update the desired state objects. + log.V(5).Infof("Applying patched templates to desired state") + if err := updateDesiredState(ctx, req, blueprint, desired); err != nil { + return errors.Wrapf(err, "failed to apply patches to desired state") + } + + return nil +} + +// createRequest creates a GenerateRequest based on the ClusterBlueprint and the desired state. +// ClusterBlueprint supplies the templates. Desired state is used to calculate variables which are later used +// as input for the patch generation. +// NOTE: GenerateRequestTemplates are created for the templates of each individual MachineDeployment in the desired +// state. This is necessary because some builtin variables are MachineDeployment specific. For example version and +// replicas of a MachineDeployment. +// NOTE: A single GenerateRequest object is used to carry templates state across subsequent Generate calls. +func createRequest(blueprint *scope.ClusterBlueprint, desired *scope.ClusterState) (*api.GenerateRequest, error) { + req := &api.GenerateRequest{} + + // Calculate global variables. + globalVariables, err := variables.Global(blueprint.Topology, desired.Cluster) + if err != nil { + return nil, errors.Wrapf(err, "failed to calculate global variables") + } + req.Variables = globalVariables + + // Add the InfrastructureClusterTemplate. + t, err := newTemplateBuilder(blueprint.InfrastructureClusterTemplate). + WithType(api.InfrastructureClusterTemplateType). + Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare InfrastructureCluster template %s for patching", + tlog.KObj{Obj: blueprint.InfrastructureClusterTemplate}) + } + req.Items = append(req.Items, t) + + // Calculate controlPlane variables. + controlPlaneVariables, err := variables.ControlPlane(&blueprint.Topology.ControlPlane, desired.ControlPlane.Object) + if err != nil { + return nil, errors.Wrapf(err, "failed to calculate ControlPlane variables") + } + + // Add the ControlPlaneTemplate. + t, err = newTemplateBuilder(blueprint.ControlPlane.Template). + WithType(api.ControlPlaneTemplateType). + Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare ControlPlane template %s for patching", + tlog.KObj{Obj: blueprint.ControlPlane.Template}) + } + t.Variables = controlPlaneVariables + req.Items = append(req.Items, t) + + // If the clusterClass mandates the controlPlane has infrastructureMachines, + // add the InfrastructureMachineTemplate for control plane machines. + if blueprint.HasControlPlaneInfrastructureMachine() { + t, err := newTemplateBuilder(blueprint.ControlPlane.InfrastructureMachineTemplate). + WithType(api.ControlPlaneInfrastructureMachineTemplateType). + Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare ControlPlane's machine template %s for patching", + tlog.KObj{Obj: blueprint.ControlPlane.InfrastructureMachineTemplate}) + } + t.Variables = controlPlaneVariables + req.Items = append(req.Items, t) + } + + // Add BootstrapConfigTemplate and InfrastructureMachine template for all MachineDeploymentTopologies + // in the Cluster. + // NOTE: We intentionally iterate over MachineDeployment in the Cluster instead of over + // MachineDeploymentClasses in the ClusterClass because each MachineDeployment in a topology + // has its own state, e.g. version or replicas. This state is used to calculate builtin variables, + // which can then be used e.g. to compute the machine image for a specific Kubernetes version. + for mdTopologyName, md := range desired.MachineDeployments { + // Lookup MachineDeploymentTopology definition from cluster.spec.topology. + mdTopology, err := lookupMDTopology(blueprint.Topology, mdTopologyName) + if err != nil { + return nil, err + } + + // Get corresponding MachineDeploymentClass from the ClusterClass. + mdClass, ok := blueprint.MachineDeployments[mdTopology.Class] + if !ok { + return nil, errors.Errorf("failed to lookup MachineDeployment class %q in ClusterClass", mdTopology.Class) + } + + // Calculate MachineDeployment variables. + mdVariables, err := variables.MachineDeployment(mdTopology, md.Object) + if err != nil { + return nil, errors.Wrapf(err, "failed to calculate variables for %s", tlog.KObj{Obj: md.Object}) + } + + // Add the BootstrapTemplate. + t, err := newTemplateBuilder(mdClass.BootstrapTemplate). + WithType(api.MachineDeploymentBootstrapConfigTemplateType). + WithMachineDeploymentRef(mdTopology). + Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare BootstrapConfig template %s for MachineDeployment topology %s for patching", + tlog.KObj{Obj: mdClass.BootstrapTemplate}, mdTopologyName) + } + t.Variables = mdVariables + req.Items = append(req.Items, t) + + // Add the InfrastructureMachineTemplate. + t, err = newTemplateBuilder(mdClass.InfrastructureMachineTemplate). + WithType(api.MachineDeploymentInfrastructureMachineTemplateType). + WithMachineDeploymentRef(mdTopology). + Build() + if err != nil { + return nil, errors.Wrapf(err, "failed to prepare InfrastructureMachine template %s for MachineDeployment topology %s for patching", + tlog.KObj{Obj: mdClass.InfrastructureMachineTemplate}, mdTopologyName) + } + t.Variables = mdVariables + req.Items = append(req.Items, t) + } + + return req, nil +} + +// lookupMDTopology looks up the MachineDeploymentTopology based on a mdTopologyName in a topology. +func lookupMDTopology(topology *clusterv1.Topology, mdTopologyName string) (*clusterv1.MachineDeploymentTopology, error) { + for _, mdTopology := range topology.Workers.MachineDeployments { + if mdTopology.Name == mdTopologyName { + return &mdTopology, nil + } + } + return nil, errors.Errorf("failed to lookup MachineDeployment topology %q in Cluster.spec.topology.workers.machineDeployments", mdTopologyName) +} + +// createPatchGenerator creates a patch generator for the given patch. +// NOTE: Currently only inline JSON patches are supported; in the future we will add +// external patches as well. +func createPatchGenerator(patch *clusterv1.ClusterClassPatch) (api.Generator, error) { + // Return a JSONPatchGenerator if there are PatchDefinitions in the patch. + if len(patch.Definitions) > 0 { + // TODO(sbueringer): will be implemented in a follow-up PR + // return inline.NewJSONPatchGenerator(patch), nil + return nil, nil + } + + return nil, errors.Errorf("failed to create patch generator for patch %q", patch.Name) +} + +// applyPatchesToRequest updates the templates of a GenerateRequest by applying the patches +// of a GenerateResponse. +func applyPatchesToRequest(ctx context.Context, req *api.GenerateRequest, resp *api.GenerateResponse) error { + log := tlog.LoggerFrom(ctx) + + for _, patch := range resp.Items { + log = log.WithValues("templateRef", templateRefString(patch.TemplateRef)) + + // Get the template the patch should be applied to. + template := getTemplate(req, patch.TemplateRef) + + // If a patch doesn't apply to any template, this is a misconfiguration. + if template == nil { + return errors.Errorf("generated patch is targeted at the template with ID %q that does not exists", templateRefString(patch.TemplateRef)) + } + + // Use the patch to create a patched copy of the template. + var patchedTemplate []byte + var err error + + switch patch.PatchType { + case api.JSONPatchType: + log.V(5).Infof("Accumulating JSON patch", "patch", string(patch.Patch.Raw)) + jsonPatch, err := jsonpatch.DecodePatch(patch.Patch.Raw) + if err != nil { + return errors.Wrapf(err, "failed to apply patch to template %s: error decoding json patch (RFC6902): %s", + templateRefString(template.TemplateRef), string(patch.Patch.Raw)) + } + + patchedTemplate, err = jsonPatch.Apply(template.Template.Raw) + if err != nil { + return errors.Wrapf(err, "failed to apply patch to template %s: error applying json patch (RFC6902): %s", + templateRefString(template.TemplateRef), string(patch.Patch.Raw)) + } + case api.JSONMergePatchType: + log.V(5).Infof("Accumulating JSON merge patch", "patch", string(patch.Patch.Raw)) + patchedTemplate, err = jsonpatch.MergePatch(template.Template.Raw, patch.Patch.Raw) + if err != nil { + return errors.Wrapf(err, "failed to apply patch to template %s: error applying json merge patch (RFC7386): %s", + templateRefString(template.TemplateRef), string(patch.Patch.Raw)) + } + } + + // Overwrite the spec of template.Template with the spec of the patchedTemplate, + // to ensure that we only pick up changes to the spec. + if err := patchTemplateSpec(&template.Template, patchedTemplate); err != nil { + return errors.Wrapf(err, "failed to apply patch to template %s", + templateRefString(template.TemplateRef)) + } + } + return nil +} + +// updateDesiredState uses the patched templates of a GenerateRequest to update the desired state. +// NOTE: This func should be called after all the patches have been applied to the GenerateRequest. +func updateDesiredState(ctx context.Context, req *api.GenerateRequest, blueprint *scope.ClusterBlueprint, desired *scope.ClusterState) error { + var err error + + // Update the InfrastructureCluster. + infrastructureClusterTemplate, err := getTemplateAsUnstructured(req, + newTemplateBuilder(blueprint.InfrastructureClusterTemplate). + WithType(api.InfrastructureClusterTemplateType). + BuildTemplateRef(), + ) + if err != nil { + return err + } + if err := patchObject(ctx, desired.InfrastructureCluster, infrastructureClusterTemplate); err != nil { + return err + } + + // Update the ControlPlane. + controlPlaneTemplate, err := getTemplateAsUnstructured(req, + newTemplateBuilder(blueprint.ControlPlane.Template). + WithType(api.ControlPlaneTemplateType). + BuildTemplateRef(), + ) + if err != nil { + return err + } + if err := patchObject(ctx, desired.ControlPlane.Object, controlPlaneTemplate, PreserveFields{ + contract.ControlPlane().MachineTemplate().Metadata().Path(), + contract.ControlPlane().MachineTemplate().InfrastructureRef().Path(), + contract.ControlPlane().MachineTemplate().NodeDrainTimeout().Path(), + contract.ControlPlane().Replicas().Path(), + contract.ControlPlane().Version().Path(), + }); err != nil { + return err + } + + // If the ClusterClass mandates the ControlPlane has InfrastructureMachines, + // update the InfrastructureMachineTemplate for ControlPlane machines. + if blueprint.HasControlPlaneInfrastructureMachine() { + infrastructureMachineTemplate, err := getTemplateAsUnstructured(req, + newTemplateBuilder(blueprint.ControlPlane.InfrastructureMachineTemplate). + WithType(api.ControlPlaneInfrastructureMachineTemplateType). + BuildTemplateRef(), + ) + if err != nil { + return err + } + if err := patchTemplate(ctx, desired.ControlPlane.InfrastructureMachineTemplate, infrastructureMachineTemplate); err != nil { + return err + } + } + + // Update the templates for all MachineDeployments. + for mdTopologyName, md := range desired.MachineDeployments { + // Lookup MachineDeploymentTopology. + mdTopology, err := lookupMDTopology(blueprint.Topology, mdTopologyName) + if err != nil { + return err + } + + // Update the BootstrapConfigTemplate. + bootstrapTemplate, err := getTemplateAsUnstructured(req, + newTemplateBuilder(md.BootstrapTemplate). + WithType(api.MachineDeploymentBootstrapConfigTemplateType). + WithMachineDeploymentRef(mdTopology). + BuildTemplateRef(), + ) + if err != nil { + return err + } + if err := patchTemplate(ctx, md.BootstrapTemplate, bootstrapTemplate); err != nil { + return err + } + + // Update the InfrastructureMachineTemplate. + infrastructureMachineTemplate, err := getTemplateAsUnstructured(req, + newTemplateBuilder(md.InfrastructureMachineTemplate). + WithType(api.MachineDeploymentInfrastructureMachineTemplateType). + WithMachineDeploymentRef(mdTopology). + BuildTemplateRef(), + ) + if err != nil { + return err + } + if err := patchTemplate(ctx, md.InfrastructureMachineTemplate, infrastructureMachineTemplate); err != nil { + return err + } + } + + return nil +} diff --git a/controllers/topology/internal/extensions/patches/engine_test.go b/controllers/topology/internal/extensions/patches/engine_test.go new file mode 100644 index 000000000000..94f6de60850a --- /dev/null +++ b/controllers/topology/internal/extensions/patches/engine_test.go @@ -0,0 +1,595 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patches + +import ( + "context" + "fmt" + "strings" + "testing" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/topology/internal/extensions/patches/api" + "sigs.k8s.io/cluster-api/controllers/topology/internal/scope" + "sigs.k8s.io/cluster-api/internal/builder" + . "sigs.k8s.io/cluster-api/internal/matchers" +) + +func TestApply(t *testing.T) { + type patch struct { + name string + patches []api.GenerateResponsePatch + } + type expectedFields struct { + infrastructureCluster map[string]interface{} + controlPlane map[string]interface{} + controlPlaneInfrastructureMachineTemplate map[string]interface{} + machineDeploymentBootstrapTemplate map[string]map[string]interface{} + machineDeploymentInfrastructureMachineTemplate map[string]map[string]interface{} + } + + tests := []struct { + name string + patches []patch + expectedFields expectedFields + }{ + { + name: "Should preserve desired state, if there are no patches", + // No changes expected. + expectedFields: expectedFields{}, + }, + { + name: "Should apply JSON patches to InfraCluster, ControlPlane and ControlPlaneInfrastructureMachineTemplate", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureClusterTemplateKind, + TemplateType: api.InfrastructureClusterTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"infraCluster"}] + `)}, + PatchType: api.JSONPatchType, + }, + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.ControlPlaneGroupVersion.String(), + Kind: builder.GenericControlPlaneTemplateKind, + TemplateType: api.ControlPlaneTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"controlPlane"}] + `)}, + PatchType: api.JSONPatchType, + }, + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureMachineTemplateKind, + TemplateType: api.ControlPlaneInfrastructureMachineTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"controlPlaneInfrastructureMachineTemplate"}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }}, + expectedFields: expectedFields{ + infrastructureCluster: map[string]interface{}{ + "spec.resource": "infraCluster", + }, + controlPlane: map[string]interface{}{ + "spec.resource": "controlPlane", + }, + controlPlaneInfrastructureMachineTemplate: map[string]interface{}{ + "spec.template.spec.resource": "controlPlaneInfrastructureMachineTemplate", + }, + }, + }, + { + name: "Should apply JSON patches to MachineDeployment templates", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { // Set /spec/template/spec/resource=topo1-infra in InfrastructureMachineTemplate of default-worker-topo1. + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureMachineTemplateKind, + TemplateType: api.MachineDeploymentInfrastructureMachineTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo1", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"topo1-infra"}] + `)}, + PatchType: api.JSONPatchType, + }, + { // Set /spec/template/spec/resource=topo1-bootstrap in BootstrapTemplate of default-worker-topo1. + TemplateRef: api.TemplateRef{ + APIVersion: builder.BootstrapGroupVersion.String(), + Kind: builder.GenericBootstrapConfigTemplateKind, + TemplateType: api.MachineDeploymentBootstrapConfigTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo1", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"topo1-bootstrap"}] + `)}, + PatchType: api.JSONPatchType, + }, + { // Set /spec/template/spec/resource=topo2-infra in InfrastructureMachineTemplate of default-worker-topo2. + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureMachineTemplateKind, + TemplateType: api.MachineDeploymentInfrastructureMachineTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo2", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"topo2-infra"}] + `)}, + PatchType: api.JSONPatchType, + }, + { // Set /spec/template/spec/resource=topo2-bootstrap in BootstrapTemplate of default-worker-topo2. + TemplateRef: api.TemplateRef{ + APIVersion: builder.BootstrapGroupVersion.String(), + Kind: builder.GenericBootstrapConfigTemplateKind, + TemplateType: api.MachineDeploymentBootstrapConfigTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo2", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"topo2-bootstrap"}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }, + }, + expectedFields: expectedFields{ + machineDeploymentBootstrapTemplate: map[string]map[string]interface{}{ + "default-worker-topo1": {"spec.template.spec.resource": "topo1-bootstrap"}, + "default-worker-topo2": {"spec.template.spec.resource": "topo2-bootstrap"}, + }, + machineDeploymentInfrastructureMachineTemplate: map[string]map[string]interface{}{ + "default-worker-topo1": {"spec.template.spec.resource": "topo1-infra"}, + "default-worker-topo2": {"spec.template.spec.resource": "topo2-infra"}, + }, + }, + }, + { + name: "Should apply same JSON patches to MachineDeployment templates", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { // Set /spec/template/spec/resource=infra in InfrastructureMachineTemplate of default-worker-topo1. + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureMachineTemplateKind, + TemplateType: api.MachineDeploymentInfrastructureMachineTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo1", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"infra"}] + `)}, + PatchType: api.JSONPatchType, + }, + { // Set /spec/template/spec/resource=infra in InfrastructureMachineTemplate of default-worker-topo2. + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureMachineTemplateKind, + TemplateType: api.MachineDeploymentInfrastructureMachineTemplateType, + MachineDeploymentRef: api.MachineDeploymentRef{ + TopologyName: "default-worker-topo2", + Class: "default-worker", + }, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/resource","value":"infra"}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }, + }, + expectedFields: expectedFields{ + machineDeploymentInfrastructureMachineTemplate: map[string]map[string]interface{}{ + "default-worker-topo1": {"spec.template.spec.resource": "infra"}, + "default-worker-topo2": {"spec.template.spec.resource": "infra"}, + }, + }, + }, + { + name: "Should apply JSON patches in the correct order", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.ControlPlaneGroupVersion.String(), + Kind: builder.GenericControlPlaneTemplateKind, + TemplateType: api.ControlPlaneTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/clusterName","value":"cluster1"}, +{"op":"add","path":"/spec/template/spec/files","value":[{"key1":"value1"}]}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }, + { + name: "fake-patch2", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.ControlPlaneGroupVersion.String(), + Kind: builder.GenericControlPlaneTemplateKind, + TemplateType: api.ControlPlaneTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"replace","path":"/spec/template/spec/clusterName","value":"cluster1-overwritten"}] +`)}, + PatchType: api.JSONPatchType, + }, + }, + }, + }, + expectedFields: expectedFields{ + controlPlane: map[string]interface{}{ + "spec.clusterName": "cluster1-overwritten", + "spec.files": []interface{}{ + map[string]interface{}{ + "key1": "value1", + }, + }, + }, + }, + }, + { + name: "Should apply JSON patches and preserve ControlPlane fields", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.ControlPlaneGroupVersion.String(), + Kind: builder.GenericControlPlaneTemplateKind, + TemplateType: api.ControlPlaneTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/replicas","value":1}, +{"op":"add","path":"/spec/template/spec/version","value":"v1.15.0"}, +{"op":"add","path":"/spec/template/spec/machineTemplate/infrastructureRef","value":{"apiVersion":"invalid","kind":"invalid","namespace":"invalid","name":"invalid"}}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }, + }, + }, + { + name: "Should apply JSON patches without metadata", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureClusterTemplateKind, + TemplateType: api.InfrastructureClusterTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +[{"op":"add","path":"/spec/template/spec/clusterName","value":"cluster1"}, +{"op":"replace","path":"/metadata/name","value": "overwrittenName"}] + `)}, + PatchType: api.JSONPatchType, + }, + }, + }, + }, + expectedFields: expectedFields{ + infrastructureCluster: map[string]interface{}{ + "spec.clusterName": "cluster1", + }, + }, + }, + { + name: "Should apply JSON merge patches", + patches: []patch{ + { + name: "fake-patch1", + patches: []api.GenerateResponsePatch{ + { + TemplateRef: api.TemplateRef{ + APIVersion: builder.InfrastructureGroupVersion.String(), + Kind: builder.GenericInfrastructureClusterTemplateKind, + TemplateType: api.InfrastructureClusterTemplateType, + }, + Patch: apiextensionsv1.JSON{Raw: []byte(` +{"spec": {"template": {"spec": {"resource": "infraCluster"}}}} + `)}, + PatchType: api.JSONMergePatchType, + }, + }, + }, + }, + expectedFields: expectedFields{ + infrastructureCluster: map[string]interface{}{ + "spec.resource": "infraCluster", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + // Set up test objects, which are: + // * blueprint: + // * A ClusterClass with its corresponding templates: + // * ControlPlaneTemplate with a corresponding ControlPlane InfrastructureMachineTemplate. + // * MachineDeploymentClass "default-worker" with corresponding BootstrapTemplate and InfrastructureMachineTemplate. + // * The corresponding Cluster.spec.topology: + // * with 3 ControlPlane replicas + // * with a "default-worker-topo1" MachineDeploymentTopology without replicas (based on "default-worker") + // * with a "default-worker-topo2" MachineDeploymentTopology with 3 replicas (based on "default-worker") + // * desired: essentially the corresponding desired objects. + blueprint, desired := setupTestObjects() + + // If there are patches, set up patch generators. + patchEngine := &engine{} + if len(tt.patches) > 0 { + for _, patch := range tt.patches { + // Add the patches to ensure the patch generator is called. + blueprint.ClusterClass.Spec.Patches = append(blueprint.ClusterClass.Spec.Patches, clusterv1.ClusterClassPatch{Name: patch.name}) + } + + patchEngine.createPatchGenerator = func(patch *clusterv1.ClusterClassPatch) (api.Generator, error) { + for _, p := range tt.patches { + if p.name == patch.Name { + return &fakePatchGenerator{patches: p.patches}, nil + } + } + return nil, errors.Errorf("could not find patch generator for patch %q", patch.Name) + } + } + + // Copy the desired objects before applying patches. + expectedCluster := desired.Cluster.DeepCopy() + expectedInfrastructureCluster := desired.InfrastructureCluster.DeepCopy() + expectedControlPlane := desired.ControlPlane.Object.DeepCopy() + expectedControlPlaneInfrastructureMachineTemplate := desired.ControlPlane.InfrastructureMachineTemplate.DeepCopy() + expectedBootstrapTemplates := map[string]*unstructured.Unstructured{} + expectedInfrastructureMachineTemplate := map[string]*unstructured.Unstructured{} + for mdTopology, md := range desired.MachineDeployments { + expectedBootstrapTemplates[mdTopology] = md.BootstrapTemplate.DeepCopy() + expectedInfrastructureMachineTemplate[mdTopology] = md.InfrastructureMachineTemplate.DeepCopy() + } + + // Set expected fields on the copy of the objects, so they can be used for comparison with the result of Apply. + if tt.expectedFields.infrastructureCluster != nil { + setSpecFields(expectedInfrastructureCluster, tt.expectedFields.infrastructureCluster) + } + if tt.expectedFields.controlPlane != nil { + setSpecFields(expectedControlPlane, tt.expectedFields.controlPlane) + } + if tt.expectedFields.controlPlaneInfrastructureMachineTemplate != nil { + setSpecFields(expectedControlPlaneInfrastructureMachineTemplate, tt.expectedFields.controlPlaneInfrastructureMachineTemplate) + } + for mdTopology, expectedFields := range tt.expectedFields.machineDeploymentBootstrapTemplate { + setSpecFields(expectedBootstrapTemplates[mdTopology], expectedFields) + } + for mdTopology, expectedFields := range tt.expectedFields.machineDeploymentInfrastructureMachineTemplate { + setSpecFields(expectedInfrastructureMachineTemplate[mdTopology], expectedFields) + } + + // Apply patches. + g.Expect(patchEngine.Apply(context.Background(), blueprint, desired)).To(Succeed()) + + // Compare the patched desired objects with the expected desired objects. + g.Expect(desired.Cluster).To(EqualObject(expectedCluster)) + g.Expect(desired.InfrastructureCluster).To(EqualObject(expectedInfrastructureCluster)) + g.Expect(desired.ControlPlane.Object).To(EqualObject(expectedControlPlane)) + g.Expect(desired.ControlPlane.InfrastructureMachineTemplate).To(EqualObject(expectedControlPlaneInfrastructureMachineTemplate)) + for mdTopology, bootstrapTemplate := range expectedBootstrapTemplates { + g.Expect(desired.MachineDeployments[mdTopology].BootstrapTemplate).To(EqualObject(bootstrapTemplate)) + } + for mdTopology, infrastructureMachineTemplate := range expectedInfrastructureMachineTemplate { + g.Expect(desired.MachineDeployments[mdTopology].InfrastructureMachineTemplate).To(EqualObject(infrastructureMachineTemplate)) + } + }) + } +} + +func setupTestObjects() (*scope.ClusterBlueprint, *scope.ClusterState) { + infrastructureClusterTemplate := builder.InfrastructureClusterTemplate(metav1.NamespaceDefault, "infraClusterTemplate1"). + Build() + + controlPlaneInfrastructureMachineTemplate := builder.InfrastructureMachineTemplate(metav1.NamespaceDefault, "controlplaneinframachinetemplate1"). + Build() + controlPlaneTemplate := builder.ControlPlaneTemplate(metav1.NamespaceDefault, "controlPlaneTemplate1"). + WithInfrastructureMachineTemplate(controlPlaneInfrastructureMachineTemplate). + Build() + + workerInfrastructureMachineTemplate := builder.InfrastructureMachineTemplate(metav1.NamespaceDefault, "linux-worker-inframachinetemplate"). + Build() + workerBootstrapTemplate := builder.BootstrapTemplate(metav1.NamespaceDefault, "linux-worker-bootstraptemplate"). + Build() + mdClass1 := builder.MachineDeploymentClass("default-worker"). + WithInfrastructureTemplate(workerInfrastructureMachineTemplate). + WithBootstrapTemplate(workerBootstrapTemplate). + Build() + + clusterClass := builder.ClusterClass(metav1.NamespaceDefault, "clusterClass1"). + WithInfrastructureClusterTemplate(infrastructureClusterTemplate). + WithControlPlaneTemplate(controlPlaneTemplate). + WithControlPlaneInfrastructureMachineTemplate(controlPlaneInfrastructureMachineTemplate). + WithWorkerMachineDeploymentClasses([]clusterv1.MachineDeploymentClass{*mdClass1}). + Build() + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: metav1.NamespaceDefault, + }, + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Version: "v1.21.2", + Class: clusterClass.Name, + ControlPlane: clusterv1.ControlPlaneTopology{ + Replicas: pointer.Int32(3), + }, + Workers: &clusterv1.WorkersTopology{ + MachineDeployments: []clusterv1.MachineDeploymentTopology{ + { + Metadata: clusterv1.ObjectMeta{}, + Class: "default-worker", + Name: "default-worker-topo1", + }, + { + Metadata: clusterv1.ObjectMeta{}, + Class: "default-worker", + Name: "default-worker-topo2", + Replicas: pointer.Int32(5), + }, + }, + }, + }, + }, + } + + // Aggregating Cluster, Templates and ClusterClass into a blueprint. + blueprint := &scope.ClusterBlueprint{ + Topology: cluster.Spec.Topology, + ClusterClass: clusterClass, + InfrastructureClusterTemplate: infrastructureClusterTemplate, + ControlPlane: &scope.ControlPlaneBlueprint{ + Template: controlPlaneTemplate, + InfrastructureMachineTemplate: controlPlaneInfrastructureMachineTemplate, + }, + MachineDeployments: map[string]*scope.MachineDeploymentBlueprint{ + "default-worker": { + InfrastructureMachineTemplate: workerInfrastructureMachineTemplate, + BootstrapTemplate: workerBootstrapTemplate, + }, + }, + } + + // Create a Cluster using the ClusterClass from above with multiple MachineDeployments + // using the same MachineDeployment class. + desiredCluster := cluster.DeepCopy() + + infrastructureCluster := builder.InfrastructureCluster(metav1.NamespaceDefault, "infraClusterTemplate1"). + WithSpecFields(map[string]interface{}{ + // Add an empty spec field, to make sure the InfrastructureCluster matches + // the one calculated by computeInfrastructureCluster. + "spec": map[string]interface{}{}, + }). + Build() + + controlPlane := builder.ControlPlane(metav1.NamespaceDefault, "controlPlane1"). + WithVersion("v1.21.2"). + WithReplicas(3). + // Make sure we're using an independent instance of the template. + WithInfrastructureMachineTemplate(controlPlaneInfrastructureMachineTemplate.DeepCopy()). + Build() + + desired := &scope.ClusterState{ + Cluster: desiredCluster, + InfrastructureCluster: infrastructureCluster, + ControlPlane: &scope.ControlPlaneState{ + Object: controlPlane, + // Make sure we're using an independent instance of the template. + InfrastructureMachineTemplate: controlPlaneInfrastructureMachineTemplate.DeepCopy(), + }, + MachineDeployments: map[string]*scope.MachineDeploymentState{ + "default-worker-topo1": { + Object: builder.MachineDeployment(metav1.NamespaceDefault, "md1"). + WithVersion("v1.21.2"). + Build(), + // Make sure we're using an independent instance of the template. + InfrastructureMachineTemplate: workerInfrastructureMachineTemplate.DeepCopy(), + BootstrapTemplate: workerBootstrapTemplate.DeepCopy(), + }, + "default-worker-topo2": { + Object: builder.MachineDeployment(metav1.NamespaceDefault, "md2"). + WithVersion("v1.20.6"). + WithReplicas(5). + Build(), + // Make sure we're using an independent instance of the template. + InfrastructureMachineTemplate: workerInfrastructureMachineTemplate.DeepCopy(), + BootstrapTemplate: workerBootstrapTemplate.DeepCopy(), + }, + }, + } + return blueprint, desired +} + +// fakePatchGenerator is an api.Generator which just returns the provided patches. +type fakePatchGenerator struct { + patches []api.GenerateResponsePatch +} + +func (g *fakePatchGenerator) Generate(_ context.Context, _ *api.GenerateRequest) (*api.GenerateResponse, error) { + return &api.GenerateResponse{ + Items: g.patches, + }, nil +} + +// setSpecFields sets fields on an unstructured object from a map. +func setSpecFields(obj *unstructured.Unstructured, fields map[string]interface{}) { + for k, v := range fields { + fieldParts := strings.Split(k, ".") + if len(fieldParts) == 0 { + panic(fmt.Errorf("fieldParts invalid")) + } + if fieldParts[0] != "spec" { + panic(fmt.Errorf("can not set fields outside spec")) + } + if err := unstructured.SetNestedField(obj.UnstructuredContent(), v, strings.Split(k, ".")...); err != nil { + panic(err) + } + } +} diff --git a/controllers/topology/internal/extensions/patches/patch.go b/controllers/topology/internal/extensions/patches/patch.go new file mode 100644 index 000000000000..89d266781a2d --- /dev/null +++ b/controllers/topology/internal/extensions/patches/patch.go @@ -0,0 +1,211 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patches + +import ( + "bytes" + "context" + "encoding/json" + "strings" + + jsonpatch "github.com/evanphx/json-patch/v5" + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/cluster-api/controllers/topology/internal/contract" + tlog "sigs.k8s.io/cluster-api/controllers/topology/internal/log" +) + +// PatchOption represents an option for the patchObject and patchTemplate funcs. +type PatchOption interface { + // ApplyToHelper applies configuration to the given options. + ApplyToHelper(*PatchOptions) +} + +// PatchOptions contains options for patchObject and patchTemplate. +type PatchOptions struct { + preserveFields []contract.Path +} + +// ApplyOptions applies the given patch options. +func (o *PatchOptions) ApplyOptions(opts []PatchOption) { + for _, opt := range opts { + opt.ApplyToHelper(o) + } +} + +// PreserveFields instructs the patch func to preserve fields. +type PreserveFields []contract.Path + +// ApplyToHelper applies this configuration to the given patch options. +func (i PreserveFields) ApplyToHelper(opts *PatchOptions) { + opts.preserveFields = i +} + +// patchObject overwrites spec in object with spec.template.spec of patchedTemplate, +// while preserving the configured fields. +// For example, ControlPlane.spec will be overwritten with the patched +// ControlPlaneTemplate.spec.template.spec but preserving spec.version and spec.replicas +// which are previously set by the topology controller and shouldn't be overwritten. +func patchObject(ctx context.Context, object, modifiedObject *unstructured.Unstructured, opts ...PatchOption) error { + return patchUnstructured(ctx, object, modifiedObject, "spec.template.spec", "spec", opts...) +} + +// patchTemplate overwrites spec.template.spec in template with spec.template.spec of patchedTemplate, +// while preserving the configured fields. +// For example, it's possible to patch BootstrapTemplate.spec.template.spec with a patched +// BootstrapTemplate.spec.template.spec while preserving fields configured via opts.fieldsToPreserve. +func patchTemplate(ctx context.Context, template, modifiedTemplate *unstructured.Unstructured, opts ...PatchOption) error { + return patchUnstructured(ctx, template, modifiedTemplate, "spec.template.spec", "spec.template.spec", opts...) +} + +// patchUnstructured overwrites original.destSpecPath with modified.srcSpecPath. +// NOTE: Original won't be changed at all, if there is no diff. +func patchUnstructured(ctx context.Context, original, modified *unstructured.Unstructured, srcSpecPath, destSpecPath string, opts ...PatchOption) error { + log := tlog.LoggerFrom(ctx) + + patchOptions := &PatchOptions{} + patchOptions.ApplyOptions(opts) + + // Create a copy to store the result of the patching temporarily. + patched := original.DeepCopy() + + // copySpec overwrites patched.destSpecPath with modified.srcSpecPath. + if err := copySpec(copySpecInput{ + src: modified, + dest: patched, + srcSpecPath: srcSpecPath, + destSpecPath: destSpecPath, + fieldsToPreserve: patchOptions.preserveFields, + }); err != nil { + return errors.Wrapf(err, "failed to apply patch to %s", tlog.KObj{Obj: original}) + } + + // Calculate diff. + diff, err := calculateDiff(original, patched) + if err != nil { + return errors.Wrapf(err, "failed to apply patch to %s: failed to calculate diff", tlog.KObj{Obj: original}) + } + + // Return if there is no diff. + if bytes.Equal(diff, []byte("{}")) { + return nil + } + + // Log the delta between the object before and after applying the accumulated patches. + log.V(4).WithObject(original).Infof("Applying accumulated patches", "diff", string(diff)) + + // Overwrite original. + *original = *patched + return nil +} + +// calculateDiff calculates the diff between two Unstructured objects. +func calculateDiff(original, patched *unstructured.Unstructured) ([]byte, error) { + originalJSON, err := json.Marshal(original.Object["spec"]) + if err != nil { + return nil, errors.Errorf("failed to marshal original object") + } + + patchedJSON, err := json.Marshal(patched.Object["spec"]) + if err != nil { + return nil, errors.Errorf("failed to marshal patched object") + } + + diff, err := jsonpatch.CreateMergePatch(originalJSON, patchedJSON) + if err != nil { + return nil, errors.Errorf("failed to diff objects") + } + return diff, nil +} + +// patchTemplateSpec overwrites spec in templateJSON with spec of patchedTemplateBytes. +func patchTemplateSpec(templateJSON *apiextensionsv1.JSON, patchedTemplateBytes []byte) error { + // Convert templates to Unstructured. + template, err := bytesToUnstructured(templateJSON.Raw) + if err != nil { + return errors.Wrap(err, "failed to convert template to Unstructured") + } + patchedTemplate, err := bytesToUnstructured(patchedTemplateBytes) + if err != nil { + return errors.Wrap(err, "failed to convert patched template to Unstructured") + } + + // Copy spec from patchedTemplate to template. + if err := copySpec(copySpecInput{ + src: patchedTemplate, + dest: template, + srcSpecPath: "spec", + destSpecPath: "spec", + }); err != nil { + return errors.Wrap(err, "failed to apply patch to template") + } + + // Marshal template and store it in templateJSON. + templateBytes, err := template.MarshalJSON() + if err != nil { + return errors.Wrapf(err, "failed to marshal patched template") + } + templateJSON.Raw = templateBytes + return nil +} + +type copySpecInput struct { + src *unstructured.Unstructured + dest *unstructured.Unstructured + srcSpecPath string + destSpecPath string + fieldsToPreserve []contract.Path +} + +// copySpec copies a field from a srcSpecPath in src to a destSpecPath in dest, +// while preserving fieldsToPreserve. +func copySpec(in copySpecInput) error { + // Backup fields that should be preserved from dest. + preservedFields := map[string]interface{}{} + for _, field := range in.fieldsToPreserve { + value, found, err := unstructured.NestedFieldNoCopy(in.dest.Object, field...) + if !found { + // Continue if the field does not exist in src. fieldsToPreserve don't have to exist. + continue + } else if err != nil { + return errors.Wrapf(err, "failed to get field %q from %s", strings.Join(field, "."), tlog.KObj{Obj: in.dest}) + } + preservedFields[strings.Join(field, ".")] = value + } + + // Get spec from src. + srcSpec, found, err := unstructured.NestedFieldNoCopy(in.src.Object, strings.Split(in.srcSpecPath, ".")...) + if !found { + return errors.Errorf("missing field %q in %s", in.srcSpecPath, tlog.KObj{Obj: in.src}) + } else if err != nil { + return errors.Wrapf(err, "failed to get field %q from %s", in.srcSpecPath, tlog.KObj{Obj: in.src}) + } + + // Set spec in dest. + if err := unstructured.SetNestedField(in.dest.Object, srcSpec, strings.Split(in.destSpecPath, ".")...); err != nil { + return errors.Wrapf(err, "failed to set field %q on %s", in.destSpecPath, tlog.KObj{Obj: in.dest}) + } + + // Restore preserved fields. + for path, value := range preservedFields { + if err := unstructured.SetNestedField(in.dest.Object, value, strings.Split(path, ".")...); err != nil { + return errors.Wrapf(err, "failed to set field %q on %s", path, tlog.KObj{Obj: in.dest}) + } + } + return nil +} diff --git a/controllers/topology/internal/extensions/patches/patch_test.go b/controllers/topology/internal/extensions/patches/patch_test.go new file mode 100644 index 000000000000..2e1b41b30e2c --- /dev/null +++ b/controllers/topology/internal/extensions/patches/patch_test.go @@ -0,0 +1,369 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patches + +import ( + "testing" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/cluster-api/controllers/topology/internal/contract" +) + +func TestCopySpec(t *testing.T) { + tests := []struct { + name string + input copySpecInput + want *unstructured.Unstructured + wantErr bool + }{ + { + name: "Field both in src and dest, no-op when equal", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + { + name: "Field both in src and dest, overwrite dest when different", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A-different", + }, + }, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + { + name: "Nested field both in src and dest, no-op when equal", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + }, + { + name: "Nested field both in src and dest, overwrite dest when different", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A-different", + }, + }, + }, + }, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + }, + { + name: "Field only in src, copy to dest", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + { + name: "Nested field only in src, copy to dest", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + srcSpecPath: "spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + }, + { + name: "Copy field from spec.template.spec in src to spec in dest", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + srcSpecPath: "spec.template.spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + { + name: "Copy field from spec.template.spec in src to spec in dest (overwrite when different)", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A-different", + }, + }, + }, + srcSpecPath: "spec.template.spec", + destSpecPath: "spec", + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + { + name: "Field both in src and dest, overwrite when different and preserve fields", + input: copySpecInput{ + src: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{ + "apiVersion": "invalid", + "kind": "invalid", + "namespace": "invalid", + "name": "invalid", + }, + }, + "replicas": float64(10), + "version": "v1.15.0", + "A": "A", + }, + }, + }, + }, + }, + dest: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{ + "apiVersion": "v1", + "kind": "kind", + "namespace": "namespace", + "name": "name", + }, + }, + "replicas": float64(3), + "version": "v1.22.0", + "A": "A-different", + }, + }, + }, + srcSpecPath: "spec.template.spec", + destSpecPath: "spec", + fieldsToPreserve: []contract.Path{ + {"spec", "machineTemplate", "infrastructureRef"}, + {"spec", "replicas"}, + {"spec", "version"}, + }, + }, + want: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{ + "apiVersion": "v1", + "kind": "kind", + "namespace": "namespace", + "name": "name", + }, + }, + "replicas": float64(3), + "version": "v1.22.0", + "A": "A", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + err := copySpec(tt.input) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(tt.input.dest).To(Equal(tt.want)) + }) + } +} diff --git a/controllers/topology/internal/extensions/patches/template.go b/controllers/topology/internal/extensions/patches/template.go new file mode 100644 index 000000000000..b5cf34b64f1f --- /dev/null +++ b/controllers/topology/internal/extensions/patches/template.go @@ -0,0 +1,165 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patches + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/topology/internal/extensions/patches/api" +) + +// templateBuilder builds templates. +type templateBuilder struct { + template *unstructured.Unstructured + templateType api.TemplateType + mdTopology *clusterv1.MachineDeploymentTopology +} + +// newTemplateBuilder returns a new templateBuilder. +func newTemplateBuilder(template *unstructured.Unstructured) *templateBuilder { + return &templateBuilder{ + template: template, + } +} + +// WithType adds templateType to the templateBuilder. +func (t *templateBuilder) WithType(templateType api.TemplateType) *templateBuilder { + t.templateType = templateType + return t +} + +// WithMachineDeploymentRef adds a MachineDeploymentTopology to the templateBuilder, +// which is used to add a MachineDeploymentRef to the TemplateRef during BuildTemplateRef. +func (t *templateBuilder) WithMachineDeploymentRef(mdTopology *clusterv1.MachineDeploymentTopology) *templateBuilder { + t.mdTopology = mdTopology + return t +} + +// BuildTemplateRef builds an api.TemplateRef. +func (t *templateBuilder) BuildTemplateRef() api.TemplateRef { + templateRef := api.TemplateRef{ + APIVersion: t.template.GetAPIVersion(), + Kind: t.template.GetKind(), + TemplateType: t.templateType, + } + + if t.mdTopology != nil { + templateRef.MachineDeploymentRef = api.MachineDeploymentRef{ + TopologyName: t.mdTopology.Name, + Class: t.mdTopology.Class, + } + } + + return templateRef +} + +// Build builds an api.GenerateRequestTemplate. +func (t *templateBuilder) Build() (*api.GenerateRequestTemplate, error) { + tpl := &api.GenerateRequestTemplate{ + TemplateRef: t.BuildTemplateRef(), + } + + jsonObj, err := json.Marshal(t.template) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal object to json") + } + tpl.Template = apiextensionsv1.JSON{Raw: jsonObj} + + return tpl, nil +} + +// getTemplateAsUnstructured is a utility func that returns a template matching the templateRef from a GenerateRequest. +func getTemplateAsUnstructured(req *api.GenerateRequest, templateRef api.TemplateRef) (*unstructured.Unstructured, error) { + // Find the template the patch should be applied to. + template := getTemplate(req, templateRef) + + // If a patch doesn't apply to any object, this is a misconfiguration. + if template == nil { + return nil, errors.Errorf("failed to get template %s", templateRefString(templateRef)) + } + + return templateToUnstructured(template) +} + +// getTemplate is a utility func that returns a template matching the templateRef from a GenerateRequest. +func getTemplate(req *api.GenerateRequest, templateRef api.TemplateRef) *api.GenerateRequestTemplate { + for _, template := range req.Items { + if templateRefsAreEqual(templateRef, template.TemplateRef) { + return template + } + } + return nil +} + +// templateRefsAreEqual returns true if the TemplateRefs are equal. +func templateRefsAreEqual(a, b api.TemplateRef) bool { + return a.APIVersion == b.APIVersion && + a.Kind == b.Kind && + a.TemplateType == b.TemplateType && + a.MachineDeploymentRef.TopologyName == b.MachineDeploymentRef.TopologyName && + a.MachineDeploymentRef.Class == b.MachineDeploymentRef.Class +} + +// templateRefString returns a string representing the TemplateRef. +func templateRefString(t api.TemplateRef) string { + ret := fmt.Sprintf("%s %s/%s", t.TemplateType, t.APIVersion, t.Kind) + if t.MachineDeploymentRef.TopologyName != "" { + ret = fmt.Sprintf("%s, MachineDeployment topology %s", ret, t.MachineDeploymentRef.TopologyName) + } + if t.MachineDeploymentRef.Class != "" { + ret = fmt.Sprintf("%s, MachineDeployment class %s", ret, t.MachineDeploymentRef.Class) + } + return ret +} + +// templateToUnstructured converts a GenerateRequestTemplate into an Unstructured object. +func templateToUnstructured(t *api.GenerateRequestTemplate) (*unstructured.Unstructured, error) { + // Unmarshal the template. + u, err := bytesToUnstructured(t.Template.Raw) + if err != nil { + return nil, errors.Wrapf(err, "failed to convert template to Unstructured") + } + + // Set the GVK. + gv, err := schema.ParseGroupVersion(t.TemplateRef.APIVersion) + if err != nil { + return nil, errors.Wrap(err, "failed to convert template to Unstructured: failed to parse group version") + } + u.SetGroupVersionKind(gv.WithKind(t.TemplateRef.Kind)) + return u, nil +} + +// bytesToUnstructured provides a utility method that converts a (JSON) byte array into an Unstructured object. +func bytesToUnstructured(b []byte) (*unstructured.Unstructured, error) { + // Unmarshal the JSON. + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal object from json") + } + + // Set the content. + u := &unstructured.Unstructured{} + u.SetUnstructuredContent(m) + + return u, nil +} diff --git a/controllers/topology/internal/extensions/patches/variables/variables.go b/controllers/topology/internal/extensions/patches/variables/variables.go new file mode 100644 index 000000000000..d4e775a6746e --- /dev/null +++ b/controllers/topology/internal/extensions/patches/variables/variables.go @@ -0,0 +1,192 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package variables calculates variables for patching. +package variables + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/topology/internal/contract" +) + +const ( + // Builtin is the prefix of all builtin variables. + // NOTE: this prefix acts as a "namespace" preventing name collision between + // user defined variable and builtin variables. + Builtin = "builtin" +) + +var ( + // Builtin Cluster variables. + + // BuiltinClusterName is the name of the cluster. + BuiltinClusterName = builtinVariable("cluster.name") + + // BuiltinClusterNamespace is the namespace of the cluster. + BuiltinClusterNamespace = builtinVariable("cluster.namespace") + + // BuiltinClusterTopologyVersion is the Kubernetes version of the Cluster. + // NOTE: Please note that this version might temporarily differ from the version + // of the ControlPlane or workers while an upgrade process is being orchestrated. + BuiltinClusterTopologyVersion = builtinVariable("cluster.topology.version") + + // BuiltinClusterTopologyClass is the name of the ClusterClass of the Cluster. + BuiltinClusterTopologyClass = builtinVariable("cluster.topology.class") + + // Builtin ControlPlane variables. + // NOTE: These variables are only set for templates belonging to the ControlPlane object. + + // BuiltinControlPlaneReplicas is the value of the replicas field of the ControlPlane object. + BuiltinControlPlaneReplicas = builtinVariable("controlPlane.replicas") + + // BuiltinControlPlaneVersion is the Kubernetes version of the ControlPlane object. + // NOTE: Please note that this version is the version we are currently reconciling towards. + // It can differ from the current version of the ControlPlane while an upgrade process is + // being orchestrated. + BuiltinControlPlaneVersion = builtinVariable("controlPlane.version") + + // Builtin MachineDeployment variables. + // NOTE: These variables are only set for templates belonging to a MachineDeployment. + + // BuiltinMachineDeploymentReplicas is the value of the replicas field of the MachineDeployment, + // to which the current template belongs to. + BuiltinMachineDeploymentReplicas = builtinVariable("machineDeployment.replicas") + + // BuiltinMachineDeploymentVersion is the Kubernetes version of the MachineDeployment, + // to which the current template belongs to. + // NOTE: Please note that this version is the version we are currently reconciling towards. + // It can differ from the current version of the MachineDeployment machines while an upgrade process is + // being orchestrated. + BuiltinMachineDeploymentVersion = builtinVariable("machineDeployment.version") + + // BuiltinMachineDeploymentClass is the class name of the MachineDeployment, + // to which the current template belongs to. + BuiltinMachineDeploymentClass = builtinVariable("machineDeployment.class") + + // BuiltinMachineDeploymentName is the name of the MachineDeployment, + // to which the current template belongs to. + BuiltinMachineDeploymentName = builtinVariable("machineDeployment.name") + + // BuiltinMachineDeploymentTopologyName is the topology name of the MachineDeployment, + // to which the current template belongs to. + BuiltinMachineDeploymentTopologyName = builtinVariable("machineDeployment.topologyName") +) + +func builtinVariable(name string) string { + return fmt.Sprintf("%s.%s", Builtin, name) +} + +// VariableMap is a name/value map of variables. +// Values are marshalled as JSON. +type VariableMap map[string]apiextensionsv1.JSON + +// Global returns variables that apply to all the templates, including user provided variables +// and builtin variables for the Cluster object. +func Global(clusterTopology *clusterv1.Topology, cluster *clusterv1.Cluster) (VariableMap, error) { + variables := VariableMap{} + + // Add user defined variables from Cluster.spec.topology.variables. + for _, variable := range clusterTopology.Variables { + variables[variable.Name] = variable.Value + } + + // Add builtin variables derived from the cluster object. + if err := setVariable(variables, BuiltinClusterName, cluster.Name); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinClusterNamespace, cluster.Namespace); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinClusterTopologyVersion, cluster.Spec.Topology.Version); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinClusterTopologyClass, cluster.Spec.Topology.Class); err != nil { + return nil, err + } + + return variables, nil +} + +// ControlPlane returns variables that apply to templates belonging to the ControlPlane. +func ControlPlane(controlPlaneTopology *clusterv1.ControlPlaneTopology, controlPlane *unstructured.Unstructured) (VariableMap, error) { + variables := VariableMap{} + + // If it is required to manage the number of replicas for the ControlPlane, set the corresponding variable. + // NOTE: If the Cluster.spec.topology.controlPlane.replicas field is nil, the topology reconciler won't set + // the replicas field on the ControlPlane. This happens either when the ControlPlane provider does + // not implement support for this field or the default value of the ControlPlane is used. + if controlPlaneTopology.Replicas != nil { + replicas, err := contract.ControlPlane().Replicas().Get(controlPlane) + if err != nil { + return nil, errors.Wrap(err, "failed to get spec.replicas from the ControlPlane") + } + if err := setVariable(variables, BuiltinControlPlaneReplicas, *replicas); err != nil { + return nil, err + } + } + + version, err := contract.ControlPlane().Version().Get(controlPlane) + if err != nil { + return nil, errors.Wrap(err, "failed to get spec.version from the ControlPlane") + } + if err := setVariable(variables, BuiltinControlPlaneVersion, version); err != nil { + return nil, err + } + + return variables, nil +} + +// MachineDeployment returns variables that apply to templates belonging to a MachineDeployment. +func MachineDeployment(mdTopology *clusterv1.MachineDeploymentTopology, md *clusterv1.MachineDeployment) (VariableMap, error) { + variables := VariableMap{} + + if md.Spec.Replicas != nil { + if err := setVariable(variables, BuiltinMachineDeploymentReplicas, *md.Spec.Replicas); err != nil { + return nil, err + } + } + if err := setVariable(variables, BuiltinMachineDeploymentVersion, *md.Spec.Template.Spec.Version); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinMachineDeploymentClass, mdTopology.Class); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinMachineDeploymentName, md.Name); err != nil { + return nil, err + } + if err := setVariable(variables, BuiltinMachineDeploymentTopologyName, mdTopology.Name); err != nil { + return nil, err + } + + return variables, nil +} + +// setVariable converts value to JSON and adds the variable to the variables map. +func setVariable(variables VariableMap, name string, value interface{}) error { + marshalledValue, err := json.Marshal(value) + if err != nil { + return errors.Wrapf(err, "failed to set variable %q: error marshalling", name) + } + + variables[name] = apiextensionsv1.JSON{Raw: marshalledValue} + return nil +} diff --git a/controllers/topology/internal/extensions/patches/variables/variables_test.go b/controllers/topology/internal/extensions/patches/variables/variables_test.go new file mode 100644 index 000000000000..edc35e770657 --- /dev/null +++ b/controllers/topology/internal/extensions/patches/variables/variables_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package variables + +import ( + "testing" + + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/internal/builder" +) + +func TestGlobal(t *testing.T) { + tests := []struct { + name string + clusterTopology *clusterv1.Topology + cluster *clusterv1.Cluster + want VariableMap + }{ + { + name: "Should calculate global variables", + clusterTopology: &clusterv1.Topology{ + Variables: []clusterv1.ClusterVariable{ + { + Name: "location", + Value: toJSON("\"us-central\""), + }, + { + Name: "cpu", + Value: toJSON("8"), + }, + { + // This is blocked by a webhook, but let's make sure we overwrite + // the user-defined variable with the builtin variable anyway. + Name: "builtin.cluster.name", + Value: toJSON("8"), + }, + }, + }, + cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: metav1.NamespaceDefault, + }, + Spec: clusterv1.ClusterSpec{ + Topology: &clusterv1.Topology{ + Class: "clusterClass1", + Version: "v1.21.1", + }, + }, + }, + want: VariableMap{ + "location": toJSON("\"us-central\""), + "cpu": toJSON("8"), + BuiltinClusterName: toJSON("\"cluster1\""), + BuiltinClusterNamespace: toJSON("\"default\""), + BuiltinClusterTopologyVersion: toJSON("\"v1.21.1\""), + BuiltinClusterTopologyClass: toJSON("\"clusterClass1\""), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := Global(tt.clusterTopology, tt.cluster) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestControlPlane(t *testing.T) { + tests := []struct { + name string + controlPlaneTopology *clusterv1.ControlPlaneTopology + controlPlane *unstructured.Unstructured + want VariableMap + }{ + { + name: "Should calculate ControlPlane variables", + controlPlaneTopology: &clusterv1.ControlPlaneTopology{ + Replicas: pointer.Int32(3), + }, + controlPlane: builder.ControlPlane(metav1.NamespaceDefault, "controlPlane1"). + WithReplicas(3). + WithVersion("v1.21.1"). + Build(), + want: VariableMap{ + BuiltinControlPlaneReplicas: toJSON("3"), + BuiltinControlPlaneVersion: toJSON("\"v1.21.1\""), + }, + }, + { + name: "Should calculate ControlPlane variables, replicas not set", + controlPlaneTopology: &clusterv1.ControlPlaneTopology{}, + controlPlane: builder.ControlPlane(metav1.NamespaceDefault, "controlPlane1"). + WithVersion("v1.21.1"). + Build(), + want: VariableMap{ + BuiltinControlPlaneVersion: toJSON("\"v1.21.1\""), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := ControlPlane(tt.controlPlaneTopology, tt.controlPlane) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestMachineDeployment(t *testing.T) { + tests := []struct { + name string + mdTopology *clusterv1.MachineDeploymentTopology + md *clusterv1.MachineDeployment + want VariableMap + }{ + { + name: "Should calculate MachineDeployment variables", + mdTopology: &clusterv1.MachineDeploymentTopology{ + Replicas: pointer.Int32(3), + Name: "md-topology", + Class: "md-class", + }, + md: builder.MachineDeployment(metav1.NamespaceDefault, "md1"). + WithReplicas(3). + WithVersion("v1.21.1"). + Build(), + want: VariableMap{ + BuiltinMachineDeploymentReplicas: toJSON("3"), + BuiltinMachineDeploymentVersion: toJSON("\"v1.21.1\""), + BuiltinMachineDeploymentClass: toJSON("\"md-class\""), + BuiltinMachineDeploymentName: toJSON("\"md1\""), + BuiltinMachineDeploymentTopologyName: toJSON("\"md-topology\""), + }, + }, + { + name: "Should calculate MachineDeployment variables, replicas not set", + mdTopology: &clusterv1.MachineDeploymentTopology{ + Name: "md-topology", + Class: "md-class", + }, + md: builder.MachineDeployment(metav1.NamespaceDefault, "md1"). + WithVersion("v1.21.1"). + Build(), + want: VariableMap{ + BuiltinMachineDeploymentVersion: toJSON("\"v1.21.1\""), + BuiltinMachineDeploymentClass: toJSON("\"md-class\""), + BuiltinMachineDeploymentName: toJSON("\"md1\""), + BuiltinMachineDeploymentTopologyName: toJSON("\"md-topology\""), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := MachineDeployment(tt.mdTopology, tt.md) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func toJSON(value string) apiextensionsv1.JSON { + return apiextensionsv1.JSON{Raw: []byte(value)} +} diff --git a/controllers/topology/internal/log/log.go b/controllers/topology/internal/log/log.go index ad3e998f8dcd..047e4e341e29 100644 --- a/controllers/topology/internal/log/log.go +++ b/controllers/topology/internal/log/log.go @@ -54,6 +54,9 @@ type Logger interface { // WithMachineDeployment adds to the logger information about the MachineDeployment object being processed. WithMachineDeployment(md *clusterv1.MachineDeployment) Logger + // WithValues adds key-value pairs of context to a logger. + WithValues(keysAndValues ...interface{}) Logger + // V returns a logger value for a specific verbosity level, relative to // this logger. V(level int) Logger @@ -105,6 +108,12 @@ func (l *topologyReconcileLogger) WithMachineDeployment(md *clusterv1.MachineDep return l } +// WithValues adds key-value pairs of context to a logger. +func (l *topologyReconcileLogger) WithValues(keysAndValues ...interface{}) Logger { + l.Logger = l.Logger.WithValues(keysAndValues...) + return l +} + // V returns a logger value for a specific verbosity level, relative to // this logger. func (l *topologyReconcileLogger) V(level int) Logger { diff --git a/docs/proposals/202105256-cluster-class-and-managed-topologies.md b/docs/proposals/202105256-cluster-class-and-managed-topologies.md index 63586cf30915..a4a919950406 100644 --- a/docs/proposals/202105256-cluster-class-and-managed-topologies.md +++ b/docs/proposals/202105256-cluster-class-and-managed-topologies.md @@ -571,11 +571,11 @@ Note: Builtin variables are defined in [Builtin variables](#builtin-variables) b #### Builtin variables It’s also possible to use so-called builtin variables in addition to user-defined variables. The following builtin variables are available: -- `cluster.{name,namespace,labels,annotations}` -- `cluster.topology.{version,class}` -- `cluster.topology.controlPlane.{labels,annotations,replicas,version}` +- `builtin.cluster.{name,namespace,labels,annotations}` +- `builtin.cluster.topology.{version,class}` +- `builtin.controlPlane.{labels,annotations,replicas,version}` - **Note**: these variables are only available when patching control plane or control plane machine templates. -- `cluster.topology.machineDeployment.current.{labels,annotations,replicas,version,class}` +- `builtin.machineDeployment.{labels,annotations,replicas,version,class,name,topologyName}` - **Note**: these variables are only available when patching MachineDeployment templates and contain the values of the current `MachineDeploymentTopology`. diff --git a/internal/builder/builders.go b/internal/builder/builders.go index 30a68e2fc8c2..dc1e9361db79 100644 --- a/internal/builder/builders.go +++ b/internal/builder/builders.go @@ -545,6 +545,8 @@ type ControlPlaneBuilder struct { namespace string name string infrastructureMachineTemplate *unstructured.Unstructured + replicas *int64 + version *string specFields map[string]interface{} statusFields map[string]interface{} } @@ -563,6 +565,18 @@ func (f *ControlPlaneBuilder) WithInfrastructureMachineTemplate(t *unstructured. return f } +// WithReplicas sets the number of replicas for the ControlPlaneBuilder. +func (f *ControlPlaneBuilder) WithReplicas(replicas int64) *ControlPlaneBuilder { + f.replicas = &replicas + return f +} + +// WithVersion adds the passed version to the ControlPlaneBuilder. +func (f *ControlPlaneBuilder) WithVersion(version string) *ControlPlaneBuilder { + f.version = &version + return f +} + // WithSpecFields sets a map of spec fields on the unstructured object. The keys in the map represent the path and the value corresponds // to the value of the spec field. // @@ -600,12 +614,23 @@ func (f *ControlPlaneBuilder) Build() *unstructured.Unstructured { setSpecFields(obj, f.specFields) setStatusFields(obj, f.statusFields) + // TODO(killianmuldoon): Update to use the internal/contract package, when it is importable from here if f.infrastructureMachineTemplate != nil { - // TODO(killianmuldoon): Update to use the internal/contract package if err := setNestedRef(obj, f.infrastructureMachineTemplate, "spec", "machineTemplate", "infrastructureRef"); err != nil { panic(err) } } + if f.replicas != nil { + if err := unstructured.SetNestedField(obj.UnstructuredContent(), *f.replicas, "spec", "replicas"); err != nil { + panic(err) + } + } + if f.version != nil { + if err := unstructured.SetNestedField(obj.UnstructuredContent(), *f.version, "spec", "version"); err != nil { + panic(err) + } + } + return obj }