diff --git a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml index 71b0f32da60a..647b9bbe55f2 100644 --- a/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml +++ b/test/e2e/data/infrastructure-docker/main/clusterclass-quick-start.yaml @@ -208,62 +208,6 @@ spec: valueFrom: template: | kindest/node:{{ .builtin.controlPlane.version | replace "+" "_" }} - - name: replaceImage-v1.23.17-machineDeployment - description: "Sets the container image for MD DockerMachineTemplates using Kubernetes v1.23.17." - enabledIf: '{{ semverCompare "v1.23.17" .builtin.machineDeployment.version }}' - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - matchResources: - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/customImage" - value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff" - - name: replaceImage-v1.23.17-controlPlane - description: "Sets the container image for CP DockerMachineTemplates using Kubernetes v1.23.17." - enabledIf: '{{ semverCompare "v1.23.17" .builtin.controlPlane.version }}' - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/customImage" - value: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff" - - name: replaceImage-v1.21.14-machineDeployment - description: "Sets the container image for MD DockerMachineTemplates using Kubernetes v1.21.14." - enabledIf: '{{ semverCompare "v1.21.14" .builtin.machineDeployment.version }}' - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - matchResources: - machineDeploymentClass: - names: - - default-worker - jsonPatches: - - op: add - path: "/spec/template/spec/customImage" - value: "kindest/node:v1.21.14@sha256:220cfafdf6e3915fbce50e13d1655425558cb98872c53f802605aa2fb2d569cf" - - name: replaceImage-v1.21.14-controlPlane - description: "Sets the container image for CP DockerMachineTemplates using Kubernetes v1.21.14." - enabledIf: '{{ semverCompare "v1.21.14" .builtin.controlPlane.version }}' - definitions: - - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: DockerMachineTemplate - matchResources: - controlPlane: true - jsonPatches: - - op: add - path: "/spec/template/spec/customImage" - value: "kindest/node:v1.21.14@sha256:220cfafdf6e3915fbce50e13d1655425558cb98872c53f802605aa2fb2d569cf" - name: preloadImages description: | Sets the container images to preload to the node that is used for running dockerMachines. diff --git a/test/extension/handlers/topologymutation/handler.go b/test/extension/handlers/topologymutation/handler.go index 7698ccbd13d0..d0f822fa8678 100644 --- a/test/extension/handlers/topologymutation/handler.go +++ b/test/extension/handlers/topologymutation/handler.go @@ -24,7 +24,6 @@ package topologymutation import ( "context" "fmt" - "strings" "github.com/blang/semver" "github.com/pkg/errors" @@ -40,6 +39,7 @@ import ( runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" "sigs.k8s.io/cluster-api/exp/runtime/topologymutation" infrav1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" "sigs.k8s.io/cluster-api/util/version" ) @@ -257,7 +257,9 @@ func patchKubeadmConfigTemplate(ctx context.Context, k *bootstrapv1.KubeadmConfi // patchDockerMachineTemplate patches the DockerMachineTemplate. // It sets the CustomImage to an image for the version in use by the controlPlane or by the MachineDeployment -// the DockerMachineTemplate belongs to. This patch is required to pick up the kind image with the required Kubernetes version. +// the DockerMachineTemplate belongs to. +// NOTE: this patch is not required anymore after the introduction of the kind mapper in kind, however we keep it +// as example of version aware patches. func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infrav1.DockerMachineTemplate, templateVariables map[string]apiextensionsv1.JSON) error { log := ctrl.LoggerFrom(ctx) @@ -270,13 +272,14 @@ func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infr return errors.Wrap(err, "could not set customImage to control plane dockerMachineTemplate") } if found { - _, err := version.ParseMajorMinorPatchTolerant(cpVersion) + semVer, err := version.ParseMajorMinorPatchTolerant(cpVersion) if err != nil { return errors.Wrap(err, "could not parse control plane version") } - customImage := fmt.Sprintf("kindest/node:%s", strings.ReplaceAll(cpVersion, "+", "_")) - log.Info(fmt.Sprintf("Setting MachineDeployment custom image to %q", customImage)) - dockerMachineTemplate.Spec.Template.Spec.CustomImage = customImage + kindMapping := kind.GetMapping(semVer, "") + + log.Info(fmt.Sprintf("Setting MachineDeployment custom image to %q", kindMapping.Image)) + dockerMachineTemplate.Spec.Template.Spec.CustomImage = kindMapping.Image // return early if we have successfully patched a control plane dockerMachineTemplate return nil } @@ -290,13 +293,14 @@ func patchDockerMachineTemplate(ctx context.Context, dockerMachineTemplate *infr return errors.Wrap(err, "could not set customImage to MachineDeployment DockerMachineTemplate") } if found { - _, err := version.ParseMajorMinorPatchTolerant(mdVersion) + semVer, err := version.ParseMajorMinorPatchTolerant(mdVersion) if err != nil { return errors.Wrap(err, "could not parse MachineDeployment version") } - customImage := fmt.Sprintf("kindest/node:%s", strings.ReplaceAll(mdVersion, "+", "_")) - log.Info(fmt.Sprintf("Setting MachineDeployment customImage to %q", customImage)) - dockerMachineTemplate.Spec.Template.Spec.CustomImage = customImage + kindMapping := kind.GetMapping(semVer, "") + + log.Info(fmt.Sprintf("Setting MachineDeployment customImage to %q", kindMapping.Image)) + dockerMachineTemplate.Spec.Template.Spec.CustomImage = kindMapping.Image return nil } diff --git a/test/infrastructure/container/docker.go b/test/infrastructure/container/docker.go index 09b09edeec86..225a0e5b9afa 100644 --- a/test/infrastructure/container/docker.go +++ b/test/infrastructure/container/docker.go @@ -39,6 +39,7 @@ import ( "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" ) const ( @@ -384,6 +385,8 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine restartMaximumRetryCount = 1 } + // TODO: check if we can simplify the following code for the CAPD load balancer, which now always has runConfig.KindMode == kind.ModeNone + hostConfig := dockercontainer.HostConfig{ // Running containers in a container requires privileges. // NOTE: we could try to replicate this with --cap-add, and use less @@ -400,6 +403,11 @@ func (d *dockerRuntime) RunContainer(ctx context.Context, runConfig *RunContaine } networkConfig := network.NetworkingConfig{} + // NOTE: starting from Kind 0.20 kind requires CgroupnsMode to be set to private. + if runConfig.KindMode != kind.ModeNone && runConfig.KindMode != kind.Mode0_19 { + hostConfig.CgroupnsMode = "private" + } + if runConfig.IPFamily == clusterv1.IPv6IPFamily { hostConfig.Sysctls = map[string]string{ "net.ipv6.conf.all.disable_ipv6": "0", diff --git a/test/infrastructure/container/interface.go b/test/infrastructure/container/interface.go index bbdfc121a4cb..2517fc1279e8 100644 --- a/test/infrastructure/container/interface.go +++ b/test/infrastructure/container/interface.go @@ -22,6 +22,7 @@ import ( "io" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" ) // providerKey is the key type for accessing the runtime provider in passed contexts. @@ -98,6 +99,8 @@ type RunContainerInput struct { // RestartPolicy to use for the container. // If not set, defaults to "unless-stopped". RestartPolicy string + // Defines how the kindest/node image must be started. + KindMode kind.Mode } // ExecContainerInput contains values for running exec on a container. diff --git a/test/infrastructure/docker/exp/internal/docker/nodepool.go b/test/infrastructure/docker/exp/internal/docker/nodepool.go index 896fe8749163..acf8ee88bb70 100644 --- a/test/infrastructure/docker/exp/internal/docker/nodepool.go +++ b/test/infrastructure/docker/exp/internal/docker/nodepool.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/blang/semver" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" @@ -37,8 +38,8 @@ import ( expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1" infraexpv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/exp/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" "sigs.k8s.io/cluster-api/util" - "sigs.k8s.io/cluster-api/util/container" ) const ( @@ -168,15 +169,18 @@ func (np *NodePool) Delete(ctx context.Context) error { } func (np *NodePool) isMachineMatchingInfrastructureSpec(machine *docker.Machine) bool { - return imageVersion(machine) == container.SemverToOCIImageTag(*np.machinePool.Spec.Template.Spec.Version) -} + // NOTE: With the current implementation we are checking if the machine is using a kindest/node image for the expected version, + // but not checking if the machine has the expected extra.mounts or pre.loaded images. + + semVer, err := semver.Parse(strings.TrimPrefix(*np.machinePool.Spec.Template.Spec.Version, "v")) + if err != nil { + // TODO: consider if to return an error + panic(errors.Wrap(err, "failed to parse DockerMachine version").Error()) + } + + kindMapping := kind.GetMapping(semVer, np.dockerMachinePool.Spec.Template.CustomImage) -// ImageVersion returns the version of the image used or nil if not specified -// NOTE: Image version might be different from the Kubernetes version, because some characters -// allowed by semver (e.g. +) can't be used for image tags, so they are replaced with "_". -func imageVersion(m *docker.Machine) string { - containerImage := m.ContainerImage() - return containerImage[strings.LastIndex(containerImage, ":")+1:] + return machine.ContainerImage() == kindMapping.Image } // machinesMatchingInfrastructureSpec returns all of the docker.Machines which match the machine pool / docker machine pool spec. diff --git a/test/infrastructure/docker/internal/docker/machine.go b/test/infrastructure/docker/internal/docker/machine.go index 34043255cdd1..6ed57f1fac0c 100644 --- a/test/infrastructure/docker/internal/docker/machine.go +++ b/test/infrastructure/docker/internal/docker/machine.go @@ -21,6 +21,7 @@ import ( "context" "encoding/base64" "fmt" + "github.com/blang/semver" "os" "path/filepath" "strings" @@ -44,27 +45,25 @@ import ( "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/cloudinit" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/provisioning/ignition" - clusterapicontainer "sigs.k8s.io/cluster-api/util/container" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" "sigs.k8s.io/cluster-api/util/patch" ) -const ( - defaultImageName = "kindest/node" - defaultImageTag = "v1.27.0" +var ( + cloudProviderTaint = corev1.Taint{Key: "node.cloudprovider.kubernetes.io/uninitialized", Effect: corev1.TaintEffectNoSchedule} ) type nodeCreator interface { - CreateControlPlaneNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error) - CreateWorkerNode(ctx context.Context, name, image, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (node *types.Node, err error) + CreateControlPlaneNode(ctx context.Context, name, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error) + CreateWorkerNode(ctx context.Context, name, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (node *types.Node, err error) } // Machine implement a service for managing the docker containers hosting a kubernetes nodes. type Machine struct { - cluster string - machine string - ipFamily clusterv1.ClusterIPFamily - container *types.Node - + cluster string + machine string + ipFamily clusterv1.ClusterIPFamily + container *types.Node nodeCreator nodeCreator } @@ -204,18 +203,26 @@ func (m *Machine) Create(ctx context.Context, image string, role string, version if m.container == nil { var err error - machineImage := m.machineImage(version) - if image != "" { - machineImage = image + // Get the KindMapping for the target K8s version. + // NOTE: The KindMapping allows to select the most recent kindest/node image available, if any, as well as + // provide info about the mode to be used when starting the kindest/node image itself. + if version == nil { + return errors.New("cannot create a DockerMachine for a nil version") + } + + semVer, err := semver.Parse(strings.TrimPrefix(*version, "v")) + if err != nil { + return errors.Wrap(err, "failed to parse DockerMachine version") } + kindMapping := kind.GetMapping(semVer, image) + switch role { case constants.ControlPlaneNodeRoleValue: - log.Info(fmt.Sprintf("Creating control plane machine container with image %s", machineImage)) + log.Info(fmt.Sprintf("Creating control plane machine container with image %s, mode %s", kindMapping.Image, kindMapping.Mode)) m.container, err = m.nodeCreator.CreateControlPlaneNode( ctx, m.ContainerName(), - machineImage, m.cluster, "127.0.0.1", 0, @@ -223,21 +230,22 @@ func (m *Machine) Create(ctx context.Context, image string, role string, version nil, labels, m.ipFamily, + kindMapping, ) if err != nil { return errors.WithStack(err) } case constants.WorkerNodeRoleValue: - log.Info(fmt.Sprintf("Creating worker machine container with image %s", machineImage)) + log.Info(fmt.Sprintf("Creating worker machine container with image %s, mode %s", kindMapping.Image, kindMapping.Mode)) m.container, err = m.nodeCreator.CreateWorkerNode( ctx, m.ContainerName(), - machineImage, m.cluster, kindMounts(mounts), nil, labels, m.ipFamily, + kindMapping, ) if err != nil { return errors.WithStack(err) @@ -452,26 +460,6 @@ func (m *Machine) Delete(ctx context.Context) error { return nil } -// machineImage is the image of the container node with the machine. -func (m *Machine) machineImage(version *string) string { - if version == nil { - defaultImage := fmt.Sprintf("%s:%s", defaultImageName, defaultImageTag) - return defaultImage - } - - // TODO(fp) make this smarter - // - allows usage of custom docker repository & image names - // - add v only for semantic versions - versionString := *version - if !strings.HasPrefix(versionString, "v") { - versionString = fmt.Sprintf("v%s", versionString) - } - - versionString = clusterapicontainer.SemverToOCIImageTag(versionString) - - return fmt.Sprintf("%s:%s", defaultImageName, versionString) -} - func logContainerDebugInfo(ctx context.Context, log logr.Logger, name string) { containerRuntime, err := container.RuntimeFrom(ctx) if err != nil { diff --git a/test/infrastructure/docker/internal/docker/kind_manager.go b/test/infrastructure/docker/internal/docker/manager.go similarity index 87% rename from test/infrastructure/docker/internal/docker/kind_manager.go rename to test/infrastructure/docker/internal/docker/manager.go index 813b4aabe4ec..91ea59727105 100644 --- a/test/infrastructure/docker/internal/docker/kind_manager.go +++ b/test/infrastructure/docker/internal/docker/manager.go @@ -27,6 +27,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/container" "sigs.k8s.io/cluster-api/test/infrastructure/docker/internal/docker/types" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" ) // KubeadmContainerPort is the port that kubeadm listens on in the container. @@ -46,7 +47,6 @@ type Manager struct{} type nodeCreateOpts struct { Name string - Image string ClusterName string Role string EntryPoint []string @@ -54,12 +54,13 @@ type nodeCreateOpts struct { PortMappings []v1alpha4.PortMapping Labels map[string]string IPFamily clusterv1.ClusterIPFamily + KindMapping kind.Mapping } // CreateControlPlaneNode will create a new control plane container. // NOTE: If port is 0 picking a host port for the control plane is delegated to the container runtime and is not stable across container restarts. // This means that connection to a control plane node may take some time to recover if the underlying container is restarted. -func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (*types.Node, error) { +func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, clusterName, listenAddress string, port int32, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (*types.Node, error) { // add api server port mapping portMappingsWithAPIServer := append(portMappings, v1alpha4.PortMapping{ ListenAddress: listenAddress, @@ -69,13 +70,13 @@ func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clust }) createOpts := &nodeCreateOpts{ Name: name, - Image: image, ClusterName: clusterName, Role: constants.ControlPlaneNodeRoleValue, PortMappings: portMappingsWithAPIServer, Mounts: mounts, Labels: labels, IPFamily: ipFamily, + KindMapping: kindMapping, } node, err := createNode(ctx, createOpts) if err != nil { @@ -86,16 +87,16 @@ func (m *Manager) CreateControlPlaneNode(ctx context.Context, name, image, clust } // CreateWorkerNode will create a new worker container. -func (m *Manager) CreateWorkerNode(ctx context.Context, name, image, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily) (*types.Node, error) { +func (m *Manager) CreateWorkerNode(ctx context.Context, name, clusterName string, mounts []v1alpha4.Mount, portMappings []v1alpha4.PortMapping, labels map[string]string, ipFamily clusterv1.ClusterIPFamily, kindMapping kind.Mapping) (*types.Node, error) { createOpts := &nodeCreateOpts{ Name: name, - Image: image, ClusterName: clusterName, Role: constants.WorkerNodeRoleValue, PortMappings: portMappings, Mounts: mounts, Labels: labels, IPFamily: ipFamily, + KindMapping: kindMapping, } return createNode(ctx, createOpts) } @@ -113,11 +114,16 @@ func (m *Manager) CreateExternalLoadBalancerNode(ctx context.Context, name, imag }} createOpts := &nodeCreateOpts{ Name: name, - Image: image, ClusterName: clusterName, Role: constants.ExternalLoadBalancerNodeRoleValue, PortMappings: portMappings, EntryPoint: haproxyEntrypoint, + // Load balancer doesn't have an equivalent in kind, but we use a kind.Mapping to + // forward the image name to create node. + KindMapping: kind.Mapping{ + Image: image, + Mode: kind.ModeNone, + }, } node, err := createNode(ctx, createOpts) if err != nil { @@ -141,7 +147,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error) runOptions := &container.RunContainerInput{ Name: opts.Name, // make hostname match container name - Image: opts.Image, + Image: opts.KindMapping.Image, Labels: containerLabels, // runtime persistent storage // this ensures that E.G. pods, logs etc. are not on the container @@ -158,6 +164,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error) "/run": "", // systemd wants a writable /run }, IPFamily: opts.IPFamily, + KindMode: opts.KindMapping.Mode, } if opts.Role == constants.ControlPlaneNodeRoleValue { runOptions.EnvironmentVars = map[string]string{ @@ -177,7 +184,7 @@ func createNode(ctx context.Context, opts *nodeCreateOpts) (*types.Node, error) return nil, err } - return types.NewNode(opts.Name, opts.Image, opts.Role), nil + return types.NewNode(opts.Name, opts.KindMapping.Image, opts.Role), nil } func generateMountInfo(mounts []v1alpha4.Mount) []container.Mount { diff --git a/test/infrastructure/docker/internal/docker/kind_manager_test.go b/test/infrastructure/docker/internal/docker/manager_test.go similarity index 88% rename from test/infrastructure/docker/internal/docker/kind_manager_test.go rename to test/infrastructure/docker/internal/docker/manager_test.go index 58bf2d86e7b5..0cc5b6a5fca8 100644 --- a/test/infrastructure/docker/internal/docker/kind_manager_test.go +++ b/test/infrastructure/docker/internal/docker/manager_test.go @@ -26,6 +26,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/infrastructure/container" + "sigs.k8s.io/cluster-api/test/infrastructure/kind" ) func TestCreateNode(t *testing.T) { @@ -44,12 +45,15 @@ func TestCreateNode(t *testing.T) { } createOpts := &nodeCreateOpts{ Name: "TestName", - Image: "TestImage", ClusterName: "TestClusterName", Role: constants.ControlPlaneNodeRoleValue, PortMappings: portMappingsWithAPIServer, Mounts: []v1alpha4.Mount{}, IPFamily: clusterv1.IPv4IPFamily, + KindMapping: kind.Mapping{ + Image: "TestImage", + Mode: kind.ModeNone, // no impact on the fake runtime. + }, } _, err := createNode(ctx, createOpts) @@ -76,7 +80,7 @@ func TestCreateControlPlaneNode(t *testing.T) { containerRuntime.ResetRunContainerCallLogs() m := Manager{} - node, err := m.CreateControlPlaneNode(ctx, "TestName", "TestImage", "TestCluster", "100.100.100.100", 80, []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily) + node, err := m.CreateControlPlaneNode(ctx, "TestName", "TestCluster", "100.100.100.100", 80, []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily, kind.Mapping{Image: "TestImage"}) g.Expect(err).ShouldNot(HaveOccurred()) g.Expect(node.Role()).Should(Equal(constants.ControlPlaneNodeRoleValue)) @@ -99,7 +103,7 @@ func TestCreateWorkerNode(t *testing.T) { containerRuntime.ResetRunContainerCallLogs() m := Manager{} - node, err := m.CreateWorkerNode(ctx, "TestName", "TestImage", "TestCluster", []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily) + node, err := m.CreateWorkerNode(ctx, "TestName", "TestCluster", []v1alpha4.Mount{}, []v1alpha4.PortMapping{}, make(map[string]string), clusterv1.IPv4IPFamily, kind.Mapping{Image: "TestImage"}) g.Expect(err).ShouldNot(HaveOccurred()) g.Expect(node.Role()).Should(Equal(constants.WorkerNodeRoleValue)) diff --git a/test/infrastructure/kind/mapper.go b/test/infrastructure/kind/mapper.go new file mode 100644 index 000000000000..411a8daf85e3 --- /dev/null +++ b/test/infrastructure/kind/mapper.go @@ -0,0 +1,312 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package kind provide utilities for CAPD interoperability with kindest/node images. + +CAPD doesn't have direct dependencies from Kind, however, it is using kindest/node images. + +As a consequence CAPD must stay in sync with Kind with regards to how kindest/node images are +run when creating a new DockerMachine. + +kindest/node images are created using a specific Kind version, and thus it is required +to keep into account the mapping between the Kubernetes version for a kindest/node and +the Kind version that created that image, so we can run it properly. + +NOTE: This is the same reason why in the Kind release documentation there is a specific list +of kindest/node images - uniquely identified by a sha - to be used with a Kind release. + +For sake of simplification, we are grouping a set of Kind versions that runs kindest/node images +in the same way into Kind mode. +*/ +package kind + +import ( + "fmt" + + "github.com/blang/semver" + + clusterapicontainer "sigs.k8s.io/cluster-api/util/container" +) + +const ( + kindestNodeImageName = "kindest/node" +) + +// Mode defines a set of Kind versions that are running images in a consistent way. +type Mode string + +const ( + // ModeNone defined a Mode to be used when creating the CAPD load balancer, which doesn't + // use a kindest/node image. + ModeNone Mode = "None" + + // Mode0_19 is a Mode that identifies kind v0.19 and older; all those versions + // are running images in a consistent way (or are too old at the time of this implementation). + Mode0_19 Mode = "kind 0.19" + + // Mode0_20 is a Mode that identifies kind v0.20 and newer; all those versions + // are running images in a consistent way, that differs from Mode0_19 because + // of the adoption of CgroupnsMode = "private" when running images. + Mode0_20 Mode = "kind 0.20" + + latestMode = Mode0_20 +) + +// Mapping defines a mapping between a Kubernetes version, a Mode, and a pre-built Image sha. +type Mapping struct { + KubernetesVersion semver.Version + Mode Mode + Image string +} + +// preBuiltMappings contains the list of kind images pre-built for a given Kind version. +// IMPORTANT: new version should be added at the beginning of the list, so in case an image for +// a given Kubernetes version is rebuilt with a newer kind version, we are using the latest image. +var preBuiltMappings = []Mapping{ + + // TODO: Add pre-built images for newer Kind versions on top + + // Pre-built images for Kind v1.20. + { + KubernetesVersion: semver.MustParse("1.27.3"), + Mode: Mode0_20, + Image: "kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72", + }, + { + KubernetesVersion: semver.MustParse("1.26.6"), + Mode: Mode0_20, + Image: "kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb", + }, + { + KubernetesVersion: semver.MustParse("1.25.11"), + Mode: Mode0_20, + Image: " kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8", + }, + { + KubernetesVersion: semver.MustParse("1.24.15"), + Mode: Mode0_20, + Image: "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab", + }, + { + KubernetesVersion: semver.MustParse("1.23.17"), + Mode: Mode0_20, + Image: "kindest/node:v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb3529d9fba91b3951d461edfdb", + }, + { + KubernetesVersion: semver.MustParse("1.22.17"), + Mode: Mode0_20, + Image: "kindest/node:v1.22.17@sha256:f5b2e5698c6c9d6d0adc419c0deae21a425c07d81bbf3b6a6834042f25d4fba2", + }, + { + KubernetesVersion: semver.MustParse("1.21.14"), + Mode: Mode0_20, + Image: "kindest/node:v1.21.14@sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093", + }, + + // Pre-built images for Kind v1.19. + { + KubernetesVersion: semver.MustParse("1.27.1"), + Mode: Mode0_19, + Image: "kindest/node:v1.27.1@sha256:b7d12ed662b873bd8510879c1846e87c7e676a79fefc93e17b2a52989d3ff42b", + }, + { + KubernetesVersion: semver.MustParse("1.26.4"), + Mode: Mode0_19, + Image: "kindest/node:v1.26.4@sha256:f4c0d87be03d6bea69f5e5dc0adb678bb498a190ee5c38422bf751541cebe92e", + }, + { + KubernetesVersion: semver.MustParse("1.25.9"), + Mode: Mode0_19, + Image: "kindest/node:v1.25.9@sha256:c08d6c52820aa42e533b70bce0c2901183326d86dcdcbedecc9343681db45161", + }, + { + KubernetesVersion: semver.MustParse("1.24.13"), + Mode: Mode0_19, + Image: "kindest/node:v1.24.13@sha256:cea86276e698af043af20143f4bf0509e730ec34ed3b7fa790cc0bea091bc5dd", + }, + { + KubernetesVersion: semver.MustParse("1.23.17"), + Mode: Mode0_19, + Image: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff", + }, + { + KubernetesVersion: semver.MustParse("1.22.17"), + Mode: Mode0_19, + Image: "kindest/node:v1.22.17@sha256:9af784f45a584f6b28bce2af84c494d947a05bd709151466489008f80a9ce9d5", + }, + { + KubernetesVersion: semver.MustParse("1.21.14"), + Mode: Mode0_19, + Image: "kindest/node:v1.21.14@sha256:220cfafdf6e3915fbce50e13d1655425558cb98872c53f802605aa2fb2d569cf", + }, + + // Pre-built and additional images for Kind v1.18. + // NOTE: This version predates the introduction of this change, but we are including it to expand + // the list of supported K8s versions; since they can be started with the same approach used up to kind 1.19, + // we are considering this version part of the Mode0_19 group. + { + KubernetesVersion: semver.MustParse("1.26.3"), + Mode: Mode0_19, + Image: "kindest/node:v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f", + }, + { + KubernetesVersion: semver.MustParse("1.25.8"), + Mode: Mode0_19, + Image: "kindest/node:v1.25.8@sha256:00d3f5314cc35327706776e95b2f8e504198ce59ac545d0200a89e69fce10b7f", + }, + { + KubernetesVersion: semver.MustParse("1.24.12"), + Mode: Mode0_19, + Image: "kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16", + }, + { + KubernetesVersion: semver.MustParse("1.23.17"), + Mode: Mode0_19, + Image: "kindest/node:v1.23.17@sha256:e5fd1d9cd7a9a50939f9c005684df5a6d145e8d695e78463637b79464292e66c", + }, + { + KubernetesVersion: semver.MustParse("1.22.17"), + Mode: Mode0_19, + Image: "kindest/node:v1.22.17@sha256:c8a828709a53c25cbdc0790c8afe12f25538617c7be879083248981945c38693", + }, + { + KubernetesVersion: semver.MustParse("1.21.14"), + Mode: Mode0_19, + Image: "kindest/node:v1.21.14@sha256:27ef72ea623ee879a25fe6f9982690a3e370c68286f4356bf643467c552a3888", + }, + { + KubernetesVersion: semver.MustParse("1.27.1"), + Mode: Mode0_19, + Image: "kindest/node:v1.27.1@sha256:9915f5629ef4d29f35b478e819249e89cfaffcbfeebda4324e5c01d53d937b09", + }, + { + KubernetesVersion: semver.MustParse("1.27.0"), + Mode: Mode0_19, + Image: "kindest/node:v1.27.0@sha256:c6b22e613523b1af67d4bc8a0c38a4c3ea3a2b8fbc5b367ae36345c9cb844518", + }, + + // Pre-built and additional images for Kind v1.17. + // NOTE: This version predates the introduction of this change, but we are including it to expand + // the list of supported K8s versions; since they can be started with the same approach used up to kind 1.19, + // we are considering this version part of the Mode0_19 group. + + { + KubernetesVersion: semver.MustParse("1.25.3"), + Mode: Mode0_19, + Image: "kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1", + }, + { + KubernetesVersion: semver.MustParse("1.24.7"), + Mode: Mode0_19, + Image: "kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315", + }, + { + KubernetesVersion: semver.MustParse("1.23.13"), + Mode: Mode0_19, + Image: "kindest/node:v1.23.13@sha256:ef453bb7c79f0e3caba88d2067d4196f427794086a7d0df8df4f019d5e336b61", + }, + { + KubernetesVersion: semver.MustParse("1.22.15"), + Mode: Mode0_19, + Image: "kindest/node:v1.22.15@sha256:7d9708c4b0873f0fe2e171e2b1b7f45ae89482617778c1c875f1053d4cef2e41", + }, + { + KubernetesVersion: semver.MustParse("1.21.14"), + Mode: Mode0_19, + Image: "kindest/node:v1.21.14@sha256:9d9eb5fb26b4fbc0c6d95fa8c790414f9750dd583f5d7cee45d92e8c26670aa1", + }, + { + KubernetesVersion: semver.MustParse("1.20.15"), + Mode: Mode0_19, + Image: "kindest/node:v1.20.15@sha256:a32bf55309294120616886b5338f95dd98a2f7231519c7dedcec32ba29699394", + }, + { + KubernetesVersion: semver.MustParse("1.19.16"), + Mode: Mode0_19, + Image: "kindest/node:v1.19.16@sha256:476cb3269232888437b61deca013832fee41f9f074f9bed79f57e4280f7c48b7", + }, + { + KubernetesVersion: semver.MustParse("1.26.0"), + Mode: Mode0_19, + Image: "kindest/node:v1.26.0@sha256:691e24bd2417609db7e589e1a479b902d2e209892a10ce375fab60a8407c7352", + }, +} + +// GetMapping return the Mapping for a given Kubernetes version/custom image. +// If a custom image is provided, return the corresponding Mapping if defined, otherwise return the mapping +// for target K8sVersion if defined; if there is no exact match for a given Kubernetes version/custom image, +// a best effort mapping is returned. +// NOTE: returning a best guess mapping is a way to try to make things to work when new images are published and +// either CAPD code in this file is not yet updated or users/CI are using older versions of CAPD. +// Even if this can lead to CAPD failing in not obvious ways, we consider this an acceptable trade off given that this +// is a provider to be used for development only and this approach allow us bit more of flexibility in using Kindest/node +// images published after CAPD version is cut (without doing code changes in the list above and/or cherry-picks). +func GetMapping(k8sVersion semver.Version, customImage string) Mapping { + bestGuess := Mapping{ + KubernetesVersion: k8sVersion, + Mode: latestMode, + Image: pickFirstNotEmpty(customImage, fallbackImage(k8sVersion)), + } + for _, m := range preBuiltMappings { + // If a custom image is provided and it matches the mapping, return it. + if m.Image == customImage { + return m + } + } + for _, m := range preBuiltMappings { + // If the mapping isn't for the right Major/Minor, ignore it. + if !(k8sVersion.Major == m.KubernetesVersion.Major && k8sVersion.Minor == m.KubernetesVersion.Minor) { + continue + } + + // If the mapping is for the same patch version, return it + if k8sVersion.Patch == m.KubernetesVersion.Patch { + return Mapping{ + KubernetesVersion: m.KubernetesVersion, + Mode: m.Mode, + Image: pickFirstNotEmpty(customImage, m.Image), + } + } + + // If the mapping is for an older patch version, then the K8s version is newer that any published image we are aware of, + // so we return out best guess. + if k8sVersion.Patch > m.KubernetesVersion.Patch { + return bestGuess + } + + // Otherwise keep looping in the existing mapping but use the oldest kind mode as a best guess. + bestGuess.Mode = m.Mode + } + return bestGuess +} + +// fallbackImage is a machine image for a given K8s version but without a sha. +// NOTE: When using fallback images there is the risk of inconsistencies between +// the kindest/node image that will be returned by the docker registry and the +// mode that will be used to start the image. +func fallbackImage(k8sVersion semver.Version) string { + versionString := clusterapicontainer.SemverToOCIImageTag(fmt.Sprintf("v%s", k8sVersion.String())) + + return fmt.Sprintf("%s:%s", kindestNodeImageName, versionString) +} + +func pickFirstNotEmpty(a, b string) string { + if a != "" { + return a + } + return b +} diff --git a/test/infrastructure/kind/mapper_test.go b/test/infrastructure/kind/mapper_test.go new file mode 100644 index 000000000000..42bf2556d3d2 --- /dev/null +++ b/test/infrastructure/kind/mapper_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "testing" + + "github.com/blang/semver" + . "github.com/onsi/gomega" +) + +func TestGetMapping(t *testing.T) { + testCases := []struct { + name string + k8sVersion semver.Version + customImage string + expectedMapping Mapping + }{ + { + name: "Exact match for custom image", + k8sVersion: semver.MustParse("1.23.17"), + customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff", + expectedMapping: Mapping{ + Mode: Mode0_19, + Image: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff", + }, + }, + { + name: "No match for custom image fallback on K8s version match", + k8sVersion: semver.MustParse("1.23.17"), + customImage: "foo", + expectedMapping: Mapping{ + Mode: Mode0_20, + Image: "foo", + }, + }, + { + name: "Exact match for Kubernetes version, kind Mode0_20", + k8sVersion: semver.MustParse("1.27.3"), + expectedMapping: Mapping{ + Mode: Mode0_20, + Image: "kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72", + }, + }, + { + name: "Exact match for Kubernetes version, kind Mode0_19", + k8sVersion: semver.MustParse("1.27.1"), + expectedMapping: Mapping{ + Mode: Mode0_19, + Image: "kindest/node:v1.27.1@sha256:b7d12ed662b873bd8510879c1846e87c7e676a79fefc93e17b2a52989d3ff42b", + }, + }, + { + name: "In case of multiple matches for Kubernetes version, return the most recent kind mode", + k8sVersion: semver.MustParse("1.23.17"), + expectedMapping: Mapping{ + Mode: Mode0_20, + Image: "kindest/node:v1.23.17@sha256:59c989ff8a517a93127d4a536e7014d28e235fb3529d9fba91b3951d461edfdb", + }, + }, + { + name: "No match Future version gets latest kind mode", + k8sVersion: semver.MustParse("1.27.99"), + expectedMapping: Mapping{ + Mode: latestMode, + Image: "kindest/node:v1.27.99", + }, + }, + { + name: "No match - In case of patch version older than the last know matches return the oldest mode know for the major/minor", + k8sVersion: semver.MustParse("1.23.0"), + expectedMapping: Mapping{ + Mode: Mode0_19, + Image: "kindest/node:v1.23.0", + }, + }, + { + name: "No Match custom image, No match Future version gets latest kind mode", + k8sVersion: semver.MustParse("1.27.99"), + customImage: "foo", + expectedMapping: Mapping{ + Mode: latestMode, + Image: "foo", + }, + }, + { + name: "No Match custom image, No match - In case of patch version older than the last know matches return the oldest mode know for the major/minor", + k8sVersion: semver.MustParse("1.23.0"), + customImage: "foo", + expectedMapping: Mapping{ + Mode: Mode0_19, + Image: "foo", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + got := GetMapping(tc.k8sVersion, tc.customImage) + + tc.expectedMapping.KubernetesVersion = tc.k8sVersion + g.Expect(got).To(Equal(tc.expectedMapping)) + }) + } +}