From c57d21e535dd544415389147a06d5dc5db3e3763 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Fri, 12 Mar 2021 10:11:24 +0100 Subject: [PATCH] stop gap for kubeadm types removal --- .../controllers/kubeadmconfig_controller.go | 8 +- .../kubeadmconfig_controller_test.go | 2 + bootstrap/kubeadm/types/v1beta1/utils.go | 60 +++++++ bootstrap/kubeadm/types/v1beta1/utils_test.go | 160 ++++++++++++++++++ .../types/v1beta2/groupversion_info.go | 26 +++ bootstrap/util/configowner.go | 14 ++ bootstrap/util/configowner_test.go | 10 ++ .../kubeadm/internal/kubeadm_config_map.go | 18 ++ .../internal/kubeadm_config_map_test.go | 31 +++- 9 files changed, 320 insertions(+), 9 deletions(-) create mode 100644 bootstrap/kubeadm/types/v1beta1/utils_test.go create mode 100644 bootstrap/kubeadm/types/v1beta2/groupversion_info.go diff --git a/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go index b14c1d6fd6b1..ca281ac7a086 100644 --- a/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go +++ b/bootstrap/kubeadm/controllers/kubeadmconfig_controller.go @@ -379,7 +379,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex }, } } - initdata, err := kubeadmv1beta1.ConfigurationToYAML(scope.Config.Spec.InitConfiguration) + initdata, err := kubeadmv1beta1.ConfigurationToYAMLForVersion(scope.Config.Spec.InitConfiguration, scope.ConfigOwner.KubernetesVersion()) if err != nil { scope.Error(err, "Failed to marshal init configuration") return ctrl.Result{}, err @@ -397,7 +397,7 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex // injects into config.ClusterConfiguration values from top level object r.reconcileTopLevelObjectSettings(scope.Cluster, machine, scope.Config) - clusterdata, err := kubeadmv1beta1.ConfigurationToYAML(scope.Config.Spec.ClusterConfiguration) + clusterdata, err := kubeadmv1beta1.ConfigurationToYAMLForVersion(scope.Config.Spec.ClusterConfiguration, scope.ConfigOwner.KubernetesVersion()) if err != nil { scope.Error(err, "Failed to marshal cluster configuration") return ctrl.Result{}, err @@ -479,7 +479,7 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) return res, nil } - joinData, err := kubeadmv1beta1.ConfigurationToYAML(scope.Config.Spec.JoinConfiguration) + joinData, err := kubeadmv1beta1.ConfigurationToYAMLForVersion(scope.Config.Spec.JoinConfiguration, scope.ConfigOwner.KubernetesVersion()) if err != nil { scope.Error(err, "Failed to marshal join configuration") return ctrl.Result{}, err @@ -560,7 +560,7 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S return res, nil } - joinData, err := kubeadmv1beta1.ConfigurationToYAML(scope.Config.Spec.JoinConfiguration) + joinData, err := kubeadmv1beta1.ConfigurationToYAMLForVersion(scope.Config.Spec.JoinConfiguration, scope.ConfigOwner.KubernetesVersion()) if err != nil { scope.Error(err, "Failed to marshal join configuration") return ctrl.Result{}, err diff --git a/bootstrap/kubeadm/controllers/kubeadmconfig_controller_test.go b/bootstrap/kubeadm/controllers/kubeadmconfig_controller_test.go index 924ae1fccb40..1193c243ac47 100644 --- a/bootstrap/kubeadm/controllers/kubeadmconfig_controller_test.go +++ b/bootstrap/kubeadm/controllers/kubeadmconfig_controller_test.go @@ -1813,6 +1813,7 @@ func newMachine(cluster *clusterv1.Cluster, name string) *clusterv1.Machine { APIVersion: bootstrapv1.GroupVersion.String(), }, }, + Version: pointer.StringPtr("v1.19.1"), }, } if cluster != nil { @@ -1854,6 +1855,7 @@ func newMachinePool(cluster *clusterv1.Cluster, name string) *expv1.MachinePool APIVersion: bootstrapv1.GroupVersion.String(), }, }, + Version: pointer.StringPtr("v1.19.1"), }, }, }, diff --git a/bootstrap/kubeadm/types/v1beta1/utils.go b/bootstrap/kubeadm/types/v1beta1/utils.go index 320aa4707dd5..2b0156e4794c 100644 --- a/bootstrap/kubeadm/types/v1beta1/utils.go +++ b/bootstrap/kubeadm/types/v1beta1/utils.go @@ -17,12 +17,72 @@ limitations under the License. package v1beta1 import ( + "strings" + "github.com/pkg/errors" runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + versionutil "k8s.io/apimachinery/pkg/util/version" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta2" "sigs.k8s.io/controller-runtime/pkg/scheme" ) +func KubeVersionToKubeadmAPIGroupVersion(version string) (schema.GroupVersion, error) { + if version == "" { + return schema.GroupVersion{}, errors.New("version cannot be empty") + } + semVersion, err := versionutil.ParseSemantic(version) + if err != nil { + return schema.GroupVersion{}, errors.Wrap(err, "error parsing the Kubernetes version") + } + switch { + case semVersion.LessThan(versionutil.MustParseSemantic("v1.13.0")): + return schema.GroupVersion{}, errors.New("the bootstrap provider for kubeadm doesn't support Kubernetes version lower than v1.13.0") + case semVersion.LessThan(versionutil.MustParseSemantic("v1.15.0")): + // NOTE: All the Kubernetes version >= v1.13 and < v1.15 should use the kubeadm API version v1beta1 + return GroupVersion, nil + default: + // NOTE: All the Kubernetes version greater or equal to v1.15 should use the kubeadm API version v1beta2. + // Also future Kubernetes versions (not yet released at the time of writing this code) are going to use v1beta2, + // no matter if kubeadm API versions newer than v1beta2 could be introduced by those release. + // This is acceptable because but v1beta2 will be supported by kubeadm until the deprecation cycle completes + // (9 months minimum after the deprecation date, not yet announced now); this gives Cluster API project time to + // introduce support for newer releases without blocking users to deploy newer version of Kubernetes. + return v1beta2.GroupVersion, nil + } +} + +// ConfigurationToYAMLForVersion converts a kubeadm configuration type to its YAML +// representation. +func ConfigurationToYAMLForVersion(obj runtime.Object, k8sVersion string) (string, error) { + yamlBytes, err := MarshalToYamlForCodecs(obj, GroupVersion, GetCodecs()) + if err != nil { + return "", errors.Wrap(err, "failed to marshal configuration") + } + + yaml := string(yamlBytes) + + // Fix the YAML according to the target Kubernetes version + // IMPORTANT: This is a stop-gap explicitly designed for back-porting on the v1alpha3 branch. + // This allows to unblock removal of the v1beta1 API in kubeadm by making Cluster API to use the v1beta2 kubeadm API + // under the assumption that the serialized version of the two APIs is equal as discussed; see + // "Insulate users from kubeadm API version changes" CAEP for more details. + // NOTE: This solution will stop to work when kubeadm will drop then v1beta2 kubeadm API, but this gives + // enough time (9/12 months from the deprecation date, not yet announced) for the users to migrate to + // the v1alpha4 release of Cluster API, where a proper conversion mechanism is going to be supported. + gv, err := KubeVersionToKubeadmAPIGroupVersion(k8sVersion) + if err != nil { + return "", err + } + + if gv != GroupVersion { + yaml = strings.Replace(yaml, GroupVersion.String(), gv.String(), -1) + } + + return yaml, nil +} + // GetCodecs returns a type that can be used to deserialize most kubeadm // configuration types. func GetCodecs() serializer.CodecFactory { diff --git a/bootstrap/kubeadm/types/v1beta1/utils_test.go b/bootstrap/kubeadm/types/v1beta1/utils_test.go new file mode 100644 index 000000000000..1f4d844821a6 --- /dev/null +++ b/bootstrap/kubeadm/types/v1beta1/utils_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta2" +) + +func Test_KubeVersionToKubeadmAPIGroupVersion(t *testing.T) { + type args struct { + k8sVersion string + } + tests := []struct { + name string + args args + want schema.GroupVersion + wantErr bool + }{ + { + name: "fails when kubernetes version is too old", + args: args{ + k8sVersion: "v1.12.0", + }, + want: schema.GroupVersion{}, + wantErr: true, + }, + { + name: "pass with minimum kubernetes version for kubeadm API v1beta1", + args: args{ + k8sVersion: "v1.13.0", + }, + want: GroupVersion, + wantErr: false, + }, + { + name: "pass with kubernetes version for kubeadm API v1beta1", + args: args{ + k8sVersion: "v1.14.99", + }, + want: GroupVersion, + wantErr: false, + }, + { + name: "pass with minimum kubernetes version for kubeadm API v1beta2", + args: args{ + k8sVersion: "v1.15.0", + }, + want: v1beta2.GroupVersion, + wantErr: false, + }, + { + name: "pass with kubernetes version for kubeadm API v1beta2", + args: args{ + k8sVersion: "v1.20.99", + }, + want: v1beta2.GroupVersion, + wantErr: false, + }, + { + name: "pass with future kubernetes version", + args: args{ + k8sVersion: "v99.99.99", + }, + want: v1beta2.GroupVersion, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := KubeVersionToKubeadmAPIGroupVersion(tt.args.k8sVersion) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func TestConfigurationToYAMLForVersion(t *testing.T) { + type args struct { + obj *ClusterConfiguration + k8sVersion string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Generates a v1beta1 kubeadm configuration with Kubernetes versions < 1.15", + args: args{ + obj: &ClusterConfiguration{}, + k8sVersion: "v1.14.9", + }, + want: "apiServer: {}\n" + + "apiVersion: kubeadm.k8s.io/v1beta1\n" + "" + + "controllerManager: {}\n" + + "dns: {}\n" + + "etcd: {}\n" + + "kind: ClusterConfiguration\n" + + "networking: {}\n" + + "scheduler: {}\n", + wantErr: false, + }, + { + name: "Generates a v1beta2 kubeadm configuration with Kubernetes versions >= 1.15", + args: args{ + obj: &ClusterConfiguration{}, + k8sVersion: "v1.15.0", + }, + want: "apiServer: {}\n" + + "apiVersion: kubeadm.k8s.io/v1beta2\n" + "" + + "controllerManager: {}\n" + + "dns: {}\n" + + "etcd: {}\n" + + "kind: ClusterConfiguration\n" + + "networking: {}\n" + + "scheduler: {}\n", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := ConfigurationToYAMLForVersion(tt.args.obj, tt.args.k8sVersion) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want), cmp.Diff(tt.want, got)) + }) + } +} diff --git a/bootstrap/kubeadm/types/v1beta2/groupversion_info.go b/bootstrap/kubeadm/types/v1beta2/groupversion_info.go new file mode 100644 index 000000000000..903a9d561fde --- /dev/null +++ b/bootstrap/kubeadm/types/v1beta2/groupversion_info.go @@ -0,0 +1,26 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "kubeadm.k8s.io", Version: "v1beta2"} +) diff --git a/bootstrap/util/configowner.go b/bootstrap/util/configowner.go index b978463013fe..8549982c6f01 100644 --- a/bootstrap/util/configowner.go +++ b/bootstrap/util/configowner.go @@ -81,6 +81,20 @@ func (co ConfigOwner) IsMachinePool() bool { return co.GetKind() == "MachinePool" } +// Returns the Kuberentes version for the config owner object +func (co ConfigOwner) KubernetesVersion() string { + fields := []string{"spec", "version"} + if co.IsMachinePool() { + fields = []string{"spec", "template", "spec", "version"} + } + + version, _, err := unstructured.NestedString(co.Object, fields...) + if err != nil { + return "" + } + return version +} + // GetConfigOwner returns the Unstructured object owning the current resource. func GetConfigOwner(ctx context.Context, c client.Client, obj metav1.Object) (*ConfigOwner, error) { allowedGKs := []schema.GroupKind{ diff --git a/bootstrap/util/configowner_test.go b/bootstrap/util/configowner_test.go index 6e1d8541b411..071507aff223 100644 --- a/bootstrap/util/configowner_test.go +++ b/bootstrap/util/configowner_test.go @@ -54,6 +54,7 @@ func TestGetConfigOwner(t *testing.T) { Bootstrap: clusterv1.Bootstrap{ DataSecretName: pointer.StringPtr("my-data-secret"), }, + Version: pointer.StringPtr("v1.19.6"), }, Status: clusterv1.MachineStatus{ InfrastructureReady: true, @@ -80,6 +81,8 @@ func TestGetConfigOwner(t *testing.T) { g.Expect(configOwner.ClusterName()).To(BeEquivalentTo("my-cluster")) g.Expect(configOwner.IsInfrastructureReady()).To(BeTrue()) g.Expect(configOwner.IsControlPlaneMachine()).To(BeTrue()) + g.Expect(configOwner.IsMachinePool()).To(BeFalse()) + g.Expect(configOwner.KubernetesVersion()).To(Equal("v1.19.6")) g.Expect(*configOwner.DataSecretName()).To(BeEquivalentTo("my-data-secret")) }) @@ -97,6 +100,11 @@ func TestGetConfigOwner(t *testing.T) { }, Spec: expv1.MachinePoolSpec{ ClusterName: "my-cluster", + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: pointer.StringPtr("v1.19.6"), + }, + }, }, Status: expv1.MachinePoolStatus{ InfrastructureReady: true, @@ -123,6 +131,8 @@ func TestGetConfigOwner(t *testing.T) { g.Expect(configOwner.ClusterName()).To(BeEquivalentTo("my-cluster")) g.Expect(configOwner.IsInfrastructureReady()).To(BeTrue()) g.Expect(configOwner.IsControlPlaneMachine()).To(BeFalse()) + g.Expect(configOwner.IsMachinePool()).To(BeTrue()) + g.Expect(configOwner.KubernetesVersion()).To(Equal("v1.19.6")) g.Expect(configOwner.DataSecretName()).To(BeNil()) }) diff --git a/controlplane/kubeadm/internal/kubeadm_config_map.go b/controlplane/kubeadm/internal/kubeadm_config_map.go index f21934b417b9..9e4a8ac17342 100644 --- a/controlplane/kubeadm/internal/kubeadm_config_map.go +++ b/controlplane/kubeadm/internal/kubeadm_config_map.go @@ -31,6 +31,7 @@ import ( const ( clusterStatusKey = "ClusterStatus" clusterConfigurationKey = "ClusterConfiguration" + apiVersionKey = "apiVersion" statusAPIEndpointsKey = "apiEndpoints" configVersionKey = "kubernetesVersion" dnsKey = "dns" @@ -90,6 +91,23 @@ func (k *kubeadmConfig) UpdateKubernetesVersion(version string) error { if err := unstructured.SetNestedField(configuration.UnstructuredContent(), version, configVersionKey); err != nil { return errors.Wrapf(err, "unable to update %q on kubeadm ConfigMap's %q", configVersionKey, clusterConfigurationKey) } + + // Fix the ClusterConfiguration according to the target Kubernetes version + // IMPORTANT: This is a stop-gap explicitly designed for back-porting on the v1alpha3 branch. + // This allows to unblock removal of the v1beta1 API in kubeadm by making Cluster API to use the v1beta2 kubeadm API + // under the assumption that the serialized version of the two APIs is equal as discussed; see + // "Insulate users from kubeadm API version changes" CAEP for more details. + // NOTE: This solution will stop to work when kubeadm will drop then v1beta2 kubeadm API, but this gives + // enough time (9/12 months from the deprecation date, not yet announced) for the users to migrate to + // the v1alpha4 release of Cluster API, where a proper conversion mechanism is going to be supported. + gv, err := kubeadmv1.KubeVersionToKubeadmAPIGroupVersion(version) + if err != nil { + return err + } + if err := unstructured.SetNestedField(configuration.UnstructuredContent(), gv.String(), apiVersionKey); err != nil { + return errors.Wrapf(err, "unable to update %q on kubeadm ConfigMap's %q", apiVersionKey, clusterConfigurationKey) + } + updated, err := yaml.Marshal(configuration) if err != nil { return errors.Wrapf(err, "unable to encode kubeadm ConfigMap's %q to YAML", clusterConfigurationKey) diff --git a/controlplane/kubeadm/internal/kubeadm_config_map_test.go b/controlplane/kubeadm/internal/kubeadm_config_map_test.go index 51516e47cf3b..6594cf5ff089 100644 --- a/controlplane/kubeadm/internal/kubeadm_config_map_test.go +++ b/controlplane/kubeadm/internal/kubeadm_config_map_test.go @@ -30,7 +30,21 @@ import ( ) func TestUpdateKubernetesVersion(t *testing.T) { - kconf := &corev1.ConfigMap{ + kconfv1beta1 := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubeadmconfig", + Namespace: metav1.NamespaceSystem, + }, + Data: map[string]string{ + clusterConfigurationKey: ` +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +kubernetesVersion: v1.16.1 +`, + }, + } + + kconfv1beta2 := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "kubeadmconfig", Namespace: metav1.NamespaceSystem, @@ -44,10 +58,10 @@ kubernetesVersion: v1.16.1 }, } - kubeadmConfigNoKey := kconf.DeepCopy() + kubeadmConfigNoKey := kconfv1beta2.DeepCopy() delete(kubeadmConfigNoKey.Data, clusterConfigurationKey) - kubeadmConfigBadData := kconf.DeepCopy() + kubeadmConfigBadData := kconfv1beta2.DeepCopy() kubeadmConfigBadData.Data[clusterConfigurationKey] = `something` tests := []struct { @@ -57,9 +71,15 @@ kubernetesVersion: v1.16.1 expectErr bool }{ { - name: "updates the config map", + name: "updates the config map and changes the kubeadm API version", + version: "v1.17.2", + config: kconfv1beta1, + expectErr: false, + }, + { + name: "updates the config map and preserves the kubeadm API version", version: "v1.17.2", - config: kconf, + config: kconfv1beta2, expectErr: false, }, { @@ -92,6 +112,7 @@ kubernetesVersion: v1.16.1 } g.Expect(err).ToNot(HaveOccurred()) g.Expect(conf.Data[clusterConfigurationKey]).To(ContainSubstring("kubernetesVersion: v1.17.2")) + g.Expect(conf.Data[clusterConfigurationKey]).To(ContainSubstring("apiVersion: kubeadm.k8s.io/v1beta2")) }) } }