diff --git a/cmd/clusterctl/client/alpha/rollout.go b/cmd/clusterctl/client/alpha/rollout.go index 4f0640f10ef3..21011188a020 100644 --- a/cmd/clusterctl/client/alpha/rollout.go +++ b/cmd/clusterctl/client/alpha/rollout.go @@ -30,6 +30,7 @@ type Rollout interface { ObjectRestarter(cluster.Proxy, util.ResourceTuple, string) error ObjectPauser(cluster.Proxy, util.ResourceTuple, string) error ObjectResumer(cluster.Proxy, util.ResourceTuple, string) error + ObjectRollbacker(cluster.Proxy, util.ResourceTuple, string, int64) error } var _ Rollout = &rollout{} diff --git a/cmd/clusterctl/client/alpha/rollout_rollbacker.go b/cmd/clusterctl/client/alpha/rollout_rollbacker.go new file mode 100644 index 000000000000..28b1badb47d2 --- /dev/null +++ b/cmd/clusterctl/client/alpha/rollout_rollbacker.go @@ -0,0 +1,171 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alpha + +import ( + "context" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster" + "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" + logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + "sigs.k8s.io/cluster-api/controllers/mdutil" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ObjectRollbacker will issue a rollback on the specified cluster-api resource. +func (r *rollout) ObjectRollbacker(proxy cluster.Proxy, tuple util.ResourceTuple, namespace string, toRevision int64) error { + switch tuple.Resource { + case "machinedeployment": + deployment, err := getMachineDeployment(proxy, tuple.Name, namespace) + if err != nil || deployment == nil { + return errors.Wrapf(err, "failed to fetch %v/%v", tuple.Resource, tuple.Name) + } + if deployment.Spec.Paused { + return errors.Errorf("can't rollback a paused machinedeployment (run rollout resume first): %v/%v\n", tuple.Resource, tuple.Name) + } + if err := rollbackMachineDeployment(proxy, deployment, tuple.Name, namespace, toRevision); err != nil { + return err + } + default: + return errors.Errorf("Invalid resource type %v. Valid values: %v", tuple.Resource, validResourceTypes) + } + return nil +} + +// rollbackMachineDeployment will rollback to a previous MachineSet revision used by this MachineDeployment. +func rollbackMachineDeployment(proxy cluster.Proxy, d *clusterv1.MachineDeployment, name, namespace string, toRevision int64) error { + log := logf.Log + c, err := proxy.NewClient() + if err != nil { + return err + } + + if toRevision < 0 { + return errors.Errorf("revision number cannot be negative: %v", toRevision) + } + msList, err := getMachineSetsForDeployment(proxy, d) + if err != nil { + return err + } + log.V(7).Info("Found MachineSets", "count", len(msList)) + msForRevision, err := findMachineDeploymentRevision(toRevision, msList) + if err != nil { + return err + } + log.V(7).Info("Found revision", "revision", msForRevision) + patchHelper, err := patch.NewHelper(d, c) + if err != nil { + return err + } + // Copy template into the machinedeployment (excluding the hash) + revMSTemplate := *msForRevision.Spec.Template.DeepCopy() + delete(revMSTemplate.Labels, mdutil.DefaultMachineDeploymentUniqueLabelKey) + + d.Spec.Template = revMSTemplate + return patchHelper.Patch(context.TODO(), d) +} + +// findMachineDeploymentRevision finds the specific revision in the machine sets +func findMachineDeploymentRevision(toRevision int64, allMSs []*clusterv1.MachineSet) (*clusterv1.MachineSet, error) { + + var ( + latestMachineSet *clusterv1.MachineSet + latestRevision = int64(-1) + previousMachineSet *clusterv1.MachineSet + previousRevision = int64(-1) + ) + for _, ms := range allMSs { + if v, err := mdutil.Revision(ms); err == nil { + if toRevision == 0 { + if latestRevision < v { + // newest one we've seen so far + previousRevision = latestRevision + previousMachineSet = latestMachineSet + latestRevision = v + latestMachineSet = ms + } else if previousRevision < v { + // second newest one we've seen so far + previousRevision = v + previousMachineSet = ms + } + } else if toRevision == v { + return ms, nil + } + } + } + + if toRevision > 0 { + return nil, errors.Errorf("unable to find specified revision: %v", toRevision) + } + + if previousMachineSet == nil { + return nil, errors.Errorf("no rollout history found") + } + return previousMachineSet, nil + +} + +// getMachineSetsForDeployment returns a list of MachineSets associated with a MachineDeployment. +func getMachineSetsForDeployment(proxy cluster.Proxy, d *clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) { + log := logf.Log + c, err := proxy.NewClient() + if err != nil { + return nil, err + } + // List all MachineSets to find those we own but that no longer match our selector. + machineSets := &clusterv1.MachineSetList{} + if err := c.List(context.TODO(), machineSets, client.InNamespace(d.Namespace)); err != nil { + return nil, err + } + + filtered := make([]*clusterv1.MachineSet, 0, len(machineSets.Items)) + for idx := range machineSets.Items { + ms := &machineSets.Items[idx] + + selector, err := metav1.LabelSelectorAsSelector(&d.Spec.Selector) + if err != nil { + log.V(5).Info("Skipping MachineSet, failed to get label selector from spec selector", "machineset", ms.Name) + continue + } + + // If a MachineDeployment with a nil or empty selector creeps in, it should match nothing, not everything. + if selector.Empty() { + log.V(5).Info("Skipping MachineSet as the selector is empty", "machineset", ms.Name) + continue + } + + // Skip this MachineSet if selector does not matche + if !selector.Matches(labels.Set(ms.Labels)) { + log.V(5).Info("Skipping MachineSet, label mismatch", "machineset", ms.Name) + continue + } + // Skip this MachineSet if its controller ref is not pointing to this MachineDeployment + if !metav1.IsControlledBy(ms, d) { + log.V(5).Info("Skipping MachineSet, controller ref does not match MachineDeployment", "machineset", ms.Name) + continue + } + + filtered = append(filtered, ms) + } + + return filtered, nil +} diff --git a/cmd/clusterctl/client/alpha/rollout_rollbacker_test.go b/cmd/clusterctl/client/alpha/rollout_rollbacker_test.go new file mode 100644 index 000000000000..9f84d83ca71c --- /dev/null +++ b/cmd/clusterctl/client/alpha/rollout_rollbacker_test.go @@ -0,0 +1,262 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alpha + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4" + "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/test" + "sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func Test_ObjectRollbacker(t *testing.T) { + labels := map[string]string{ + clusterv1.ClusterLabelName: "test", + clusterv1.MachineDeploymentLabelName: "test-md-0", + } + currentVersion := "v1.19.3" + rollbackVersion := "v1.19.1" + deployment := &clusterv1.MachineDeployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "MachineDeployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-md-0", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + Annotations: map[string]string{ + clusterv1.RevisionAnnotation: "2", + }, + }, + Spec: clusterv1.MachineDeploymentSpec{ + ClusterName: "test", + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + }, + Template: clusterv1.MachineTemplateSpec{ + ObjectMeta: clusterv1.ObjectMeta{ + Labels: labels, + }, + Spec: clusterv1.MachineSpec{ + ClusterName: "test", + Version: ¤tVersion, + InfrastructureRef: corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha4", + Kind: "InfrastructureMachineTemplate", + Name: "md-template", + }, + Bootstrap: clusterv1.Bootstrap{ + DataSecretName: pointer.StringPtr("data-secret-name"), + }, + }, + }, + }, + } + type fields struct { + objs []client.Object + tuple util.ResourceTuple + namespace string + toRevision int64 + } + tests := []struct { + name string + fields fields + wantErr bool + wantVersion string + wantInfraTemplate string + wantBootsrapSecretName string + }{ + { + name: "machinedeployment should rollback to revision=1", + fields: fields{ + objs: []client.Object{ + deployment, + &clusterv1.MachineSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ms-rev-2", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, clusterv1.GroupVersion.WithKind("MachineDeployment")), + }, + Labels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + Annotations: map[string]string{ + clusterv1.RevisionAnnotation: "2", + }, + }, + }, + &clusterv1.MachineSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "ms-rev-1", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, clusterv1.GroupVersion.WithKind("MachineDeployment")), + }, + Labels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + Annotations: map[string]string{ + clusterv1.RevisionAnnotation: "1", + }, + }, + Spec: clusterv1.MachineSetSpec{ + ClusterName: "test", + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + }, + Template: clusterv1.MachineTemplateSpec{ + ObjectMeta: clusterv1.ObjectMeta{ + Labels: labels, + }, + Spec: clusterv1.MachineSpec{ + ClusterName: "test", + Version: &rollbackVersion, + InfrastructureRef: corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha4", + Kind: "InfrastructureMachineTemplate", + Name: "md-template-rollback", + }, + Bootstrap: clusterv1.Bootstrap{ + DataSecretName: pointer.StringPtr("data-secret-name-rollback"), + }, + }, + }, + }, + }, + }, + tuple: util.ResourceTuple{ + Resource: "machinedeployment", + Name: "test-md-0", + }, + namespace: "default", + toRevision: int64(1), + }, + wantErr: false, + wantVersion: rollbackVersion, + wantInfraTemplate: "md-template-rollback", + wantBootsrapSecretName: "data-secret-name-rollback", + }, + { + name: "machinedeployment should not rollback because there is no previous revision", + fields: fields{ + objs: []client.Object{ + deployment, + &clusterv1.MachineSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ms-rev-2", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, clusterv1.GroupVersion.WithKind("MachineDeployment")), + }, + Labels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + Annotations: map[string]string{ + clusterv1.RevisionAnnotation: "2", + }, + }, + }, + }, + tuple: util.ResourceTuple{ + Resource: "machinedeployment", + Name: "test-md-0", + }, + namespace: "default", + toRevision: int64(0), + }, + wantErr: true, + }, + { + name: "machinedeployment should not rollback because the specified version does not exist", + fields: fields{ + objs: []client.Object{ + deployment, + &clusterv1.MachineSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ms-rev-2", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, clusterv1.GroupVersion.WithKind("MachineDeployment")), + }, + Labels: map[string]string{ + clusterv1.ClusterLabelName: "test", + }, + Annotations: map[string]string{ + clusterv1.RevisionAnnotation: "2", + }, + }, + }, + }, + tuple: util.ResourceTuple{ + Resource: "machinedeployment", + Name: "test-md-0", + }, + namespace: "default", + toRevision: int64(1), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + r := newRolloutClient() + proxy := test.NewFakeProxy().WithObjs(tt.fields.objs...) + err := r.ObjectRollbacker(proxy, tt.fields.tuple, tt.fields.namespace, tt.fields.toRevision) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + cl, err := proxy.NewClient() + g.Expect(err).ToNot(HaveOccurred()) + key := client.ObjectKeyFromObject(deployment) + md := &clusterv1.MachineDeployment{} + err = cl.Get(context.TODO(), key, md) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(*md.Spec.Template.Spec.Version).To(Equal(tt.wantVersion)) + g.Expect(md.Spec.Template.Spec.InfrastructureRef.Name).To(Equal(tt.wantInfraTemplate)) + g.Expect(*md.Spec.Template.Spec.Bootstrap.DataSecretName).To(Equal(tt.wantBootsrapSecretName)) + }) + } +} diff --git a/cmd/clusterctl/client/client.go b/cmd/clusterctl/client/client.go index 8e7d5def5c63..6fd0f7222964 100644 --- a/cmd/clusterctl/client/client.go +++ b/cmd/clusterctl/client/client.go @@ -83,6 +83,8 @@ type AlphaClient interface { RolloutPause(options RolloutOptions) error // RolloutResume provides rollout resume of paused cluster-api resources RolloutResume(options RolloutOptions) error + // RolloutUndo provides rollout rollback of cluster-api resources + RolloutUndo(options RolloutOptions) error } // YamlPrinter exposes methods that prints the processed template and diff --git a/cmd/clusterctl/client/rollout.go b/cmd/clusterctl/client/rollout.go index 7bd5c66e4432..7e5c735c8237 100644 --- a/cmd/clusterctl/client/rollout.go +++ b/cmd/clusterctl/client/rollout.go @@ -36,6 +36,10 @@ type RolloutOptions struct { // Namespace where the resource(s) live. If unspecified, the namespace name will be inferred // from the current configuration. Namespace string + + // Revision number to rollback to when issuing the undo command. + // Revision number of a specific revision when issuing the history command. + ToRevision int64 } func (c *clusterctlClient) RolloutRestart(options RolloutOptions) error { @@ -110,6 +114,38 @@ func getResourceTuples(clusterClient cluster.Client, options RolloutOptions) ([] return tuples, nil } +func (c *clusterctlClient) RolloutUndo(options RolloutOptions) error { + clusterClient, err := c.clusterClientFactory(ClusterClientFactoryInput{Kubeconfig: options.Kubeconfig}) + if err != nil { + return err + } + + // If the option specifying the Namespace is empty, try to detect it. + if options.Namespace == "" { + currentNamespace, err := clusterClient.Proxy().CurrentNamespace() + if err != nil { + return err + } + options.Namespace = currentNamespace + } + + if len(options.Resources) == 0 { + return fmt.Errorf("required resource not specified") + } + normalized := normalizeResources(options.Resources) + tuples, err := util.ResourceTypeAndNameArgs(normalized...) + if err != nil { + return err + } + + for _, t := range tuples { + if err := c.alphaClient.Rollout().ObjectRollbacker(clusterClient.Proxy(), t, options.Namespace, options.ToRevision); err != nil { + return err + } + } + return nil +} + func normalizeResources(input []string) []string { normalized := make([]string, 0, len(input)) for _, in := range input { diff --git a/cmd/clusterctl/cmd/rollout.go b/cmd/clusterctl/cmd/rollout.go index 678f0907fa2d..3d3b874d6209 100644 --- a/cmd/clusterctl/cmd/rollout.go +++ b/cmd/clusterctl/cmd/rollout.go @@ -36,8 +36,11 @@ var ( # Mark the machinedeployment as paused clusterctl alpha rollout pause machinedeployment/my-md-0 - # Resume an already paused deployment - clusterctl alpha rollout resume machinedeployment/my-md-0`) + # Resume an already paused machinedeployment + clusterctl alpha rollout resume machinedeployment/my-md-0 + + # Rollback a machinedeployment + clusterctl alpha rollout undo machinedeployment/my-md-0 --to-revision=3`) rolloutCmd = &cobra.Command{ Use: "rollout SUBCOMMAND", @@ -52,4 +55,5 @@ func init() { rolloutCmd.AddCommand(rollout.NewCmdRolloutRestart(cfgFile)) rolloutCmd.AddCommand(rollout.NewCmdRolloutPause(cfgFile)) rolloutCmd.AddCommand(rollout.NewCmdRolloutResume(cfgFile)) + rolloutCmd.AddCommand(rollout.NewCmdRolloutUndo(cfgFile)) } diff --git a/cmd/clusterctl/cmd/rollout/undo.go b/cmd/clusterctl/cmd/rollout/undo.go new file mode 100644 index 000000000000..ba8a63cf57b8 --- /dev/null +++ b/cmd/clusterctl/cmd/rollout/undo.go @@ -0,0 +1,88 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollout + +import ( + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + "sigs.k8s.io/cluster-api/cmd/clusterctl/client" +) + +// undoOptions is the start of the data required to perform the operation. +type undoOptions struct { + kubeconfig string + kubeconfigContext string + resources []string + namespace string + toRevision int64 +} + +var undoOpt = &undoOptions{} + +var ( + undoLong = templates.LongDesc(` + Rollback to a previous rollout.`) + + undoExample = templates.Examples(` + # Rollback to the previous deployment + clusterctl alpha rollout undo machinedeployment/my-md-0 + + # Rollback to previous machinedeployment --to-revision=3 + clusterctl alpha rollout undo machinedeployment/my-md-0 --to-revision=3`) +) + +// NewCmdRolloutUndo returns a Command instance for 'rollout undo' sub command +func NewCmdRolloutUndo(cfgFile string) *cobra.Command { + + cmd := &cobra.Command{ + Use: "undo RESOURCE", + DisableFlagsInUseLine: true, + Short: "Undo a cluster-api resource", + Long: undoLong, + Example: undoExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runUndo(cfgFile, cmd, args) + }, + } + cmd.Flags().StringVar(&undoOpt.kubeconfig, "kubeconfig", "", + "Path to the kubeconfig file to use for accessing the management cluster. If unspecified, default discovery rules apply.") + cmd.Flags().StringVar(&undoOpt.kubeconfigContext, "kubeconfig-context", "", + "Context to be used within the kubeconfig file. If empty, current context will be used.") + cmd.Flags().StringVar(&undoOpt.namespace, "namespace", "", "Namespace where the resource(s) reside. If unspecified, the defult namespace will be used.") + cmd.Flags().Int64Var(&undoOpt.toRevision, "to-revision", undoOpt.toRevision, "The revision to rollback to. Default to 0 (last revision).") + + return cmd +} + +func runUndo(cfgFile string, cmd *cobra.Command, args []string) error { + undoOpt.resources = args + + c, err := client.New(cfgFile) + if err != nil { + return err + } + + if err := c.RolloutUndo(client.RolloutOptions{ + Kubeconfig: client.Kubeconfig{Path: undoOpt.kubeconfig, Context: undoOpt.kubeconfigContext}, + Namespace: undoOpt.namespace, + Resources: undoOpt.resources, + ToRevision: undoOpt.toRevision, + }); err != nil { + return err + } + return nil +}