diff --git a/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go b/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go index 9bdd7a9fc9c5..bc4fcf4a38d0 100644 --- a/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go +++ b/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go @@ -1019,17 +1019,16 @@ func GetClusterAPIObject(client Client, clusterName, namespace string) (*cluster return nil, nil, nil, errors.Wrapf(err, "unable to fetch cluster %s/%s", namespace, clusterName) } - controlPlane, nodes, err := ExtractControlPlaneMachine(machines) + controlPlane, nodes, err := ExtractControlPlaneMachines(machines) if err != nil { return nil, nil, nil, errors.Wrapf(err, "unable to fetch control plane machine in cluster %s/%s", namespace, clusterName) } - return cluster, controlPlane, nodes, nil + return cluster, controlPlane[0], nodes, nil } -// ExtractControlPlaneMachine separates the machines running the control plane (singular) from the incoming machines. +// ExtractControlPlaneMachines separates the machines running the control plane from the incoming machines. // This is currently done by looking at which machine specifies the control plane version. -// TODO: Cleanup. -func ExtractControlPlaneMachine(machines []*clusterv1.Machine) (*clusterv1.Machine, []*clusterv1.Machine, error) { +func ExtractControlPlaneMachines(machines []*clusterv1.Machine) ([]*clusterv1.Machine, []*clusterv1.Machine, error) { nodes := []*clusterv1.Machine{} controlPlaneMachines := []*clusterv1.Machine{} for _, machine := range machines { @@ -1039,8 +1038,8 @@ func ExtractControlPlaneMachine(machines []*clusterv1.Machine) (*clusterv1.Machi nodes = append(nodes, machine) } } - if len(controlPlaneMachines) != 1 { - return nil, nil, errors.Errorf("expected one control plane machine, got: %v", len(controlPlaneMachines)) + if len(controlPlaneMachines) < 1 { + return nil, nil, errors.Errorf("expected one or more control plane machines, got: %v", len(controlPlaneMachines)) } - return controlPlaneMachines[0], nodes, nil + return controlPlaneMachines, nodes, nil } diff --git a/cmd/clusterctl/clusterdeployer/clusterdeployer.go b/cmd/clusterctl/clusterdeployer/clusterdeployer.go index 73d17b3e09c5..f3263b508c7d 100644 --- a/cmd/clusterctl/clusterdeployer/clusterdeployer.go +++ b/cmd/clusterctl/clusterdeployer/clusterdeployer.go @@ -57,7 +57,7 @@ func New( // Create the cluster from the provided cluster definition and machine list. func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*clusterv1.Machine, provider provider.Deployer, kubeconfigOutput string, providerComponentsStoreFactory provider.ComponentsStoreFactory) error { - controlPlaneMachine, nodes, err := clusterclient.ExtractControlPlaneMachine(machines) + controlPlaneMachines, nodes, err := clusterclient.ExtractControlPlaneMachines(machines) if err != nil { return errors.Wrap(err, "unable to separate control plane machines from node machines") } @@ -89,12 +89,12 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster cluster.Namespace = bootstrapClient.GetContextNamespace() } - klog.Infof("Creating control plane %v in namespace %q", controlPlaneMachine.Name, cluster.Namespace) - if err := phases.ApplyMachines(bootstrapClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachine}); err != nil { + klog.Infof("Creating control plane %v in namespace %q", controlPlaneMachines[0].Name, cluster.Namespace) + if err := phases.ApplyMachines(bootstrapClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachines[0]}); err != nil { return errors.Wrap(err, "unable to create control plane machine") } - klog.Infof("Updating bootstrap cluster object for cluster %v in namespace %q with control plane endpoint running on %s", cluster.Name, cluster.Namespace, controlPlaneMachine.Name) + klog.Infof("Updating bootstrap cluster object for cluster %v in namespace %q with control plane endpoint running on %s", cluster.Name, cluster.Namespace, controlPlaneMachines[0].Name) if err := d.updateClusterEndpoint(bootstrapClient, provider, cluster.Name, cluster.Namespace); err != nil { return errors.Wrap(err, "unable to update bootstrap cluster endpoint") } @@ -130,11 +130,22 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster // For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we // update the target cluster endpoint as well to be sure. - klog.Infof("Updating target cluster object with control plane endpoint running on %s", controlPlaneMachine.Name) + klog.Infof("Updating target cluster object with control plane endpoint running on %s", controlPlaneMachines[0].Name) if err := d.updateClusterEndpoint(targetClient, provider, cluster.Name, cluster.Namespace); err != nil { return errors.Wrap(err, "unable to update target cluster endpoint") } + if len(controlPlaneMachines) > 1 { + // TODO(h0tbird) Done serially until kubernetes/kubeadm#1097 is resolved and all + // supported versions of k8s we are deploying (using kubeadm) have the fix. + klog.Info("Creating additional control plane machines in target cluster.") + for _, controlPlaneMachine := range controlPlaneMachines[1:] { + if err := phases.ApplyMachines(targetClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachine}); err != nil { + return errors.Wrap(err, "unable to create additional control plane machines") + } + } + } + klog.Info("Creating node machines in target cluster.") if err := phases.ApplyMachines(targetClient, cluster.Namespace, nodes); err != nil { return errors.Wrap(err, "unable to create node machines") diff --git a/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go b/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go index 8007b39e38ea..1a9e39e1a254 100644 --- a/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go +++ b/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go @@ -660,17 +660,6 @@ func TestClusterCreate(t *testing.T) { expectExternalExists: false, expectExternalCreated: true, }, - { - name: "fail provision multiple clusters in a namespace", - targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, - bootstrapClient: &testClusterClient{}, - namespaceToExpectedInternalMachines: make(map[string]int), - namespaceToInputCluster: map[string][]*clusterv1.Cluster{"foo": getClustersForNamespace("foo", 3)}, - expectErr: true, - cleanupExternal: true, - expectExternalExists: false, - expectExternalCreated: true, - }, { name: "fail provision bootstrap cluster", targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, @@ -996,42 +985,49 @@ func TestExtractControlPlaneMachine(t *testing.T) { testCases := []struct { name string inputMachines []*clusterv1.Machine - expectedControlPlane *clusterv1.Machine - expectedNodes []*clusterv1.Machine + expectedCPMachines []*clusterv1.Machine + expectedNodeMachines []*clusterv1.Machine expectedError error }{ { name: "success_1_control_plane_1_node", inputMachines: generateMachines(nil, metav1.NamespaceDefault), - expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName), - expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}), + expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}), + expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}), expectedError: nil, }, { name: "success_1_control_plane_multiple_nodes", - inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, singleControlPlaneName, multipleNodeNames), - expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName), - expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), + inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}, multipleNodeNames), + expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}), + expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), expectedError: nil, }, { - name: "fail_more_than_1_control_plane_not_allowed", - inputMachines: generateInvalidExtractControlPlaneMachine(nil, metav1.NamespaceDefault, multipleControlPlaneNames, multipleNodeNames), - expectedControlPlane: nil, - expectedNodes: nil, - expectedError: errors.New("expected one control plane machine, got: 2"), + name: "success_2_control_planes_1_node", + inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, multipleControlPlaneNames, []string{singleNodeName}), + expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, multipleControlPlaneNames), + expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}), + expectedError: nil, + }, + { + name: "success_2_control_planes_multiple_nodes", + inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, multipleControlPlaneNames, multipleNodeNames), + expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, multipleControlPlaneNames), + expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), + expectedError: nil, }, { name: "fail_0_control_plane_not_allowed", inputMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), - expectedControlPlane: nil, - expectedNodes: nil, - expectedError: errors.New("expected one control plane machine, got: 0"), + expectedCPMachines: nil, + expectedNodeMachines: nil, + expectedError: errors.New("expected one or more control plane machines, got: 0"), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actualControlPlane, actualNodes, actualError := clusterclient.ExtractControlPlaneMachine(tc.inputMachines) + actualCPMachines, actualNodes, actualError := clusterclient.ExtractControlPlaneMachines(tc.inputMachines) if tc.expectedError == nil && actualError != nil { t.Fatalf("%s: extractControlPlaneMachine(%q): gotError %q; wantError [nil]", tc.name, len(tc.inputMachines), actualError) @@ -1041,13 +1037,13 @@ func TestExtractControlPlaneMachine(t *testing.T) { t.Fatalf("%s: extractControlPlaneMachine(%q): gotError %q; wantError %q", tc.name, len(tc.inputMachines), actualError, tc.expectedError) } - if (tc.expectedControlPlane == nil && actualControlPlane != nil) || - (tc.expectedControlPlane != nil && actualControlPlane == nil) { - t.Fatalf("%s: extractControlPlaneMachine(%q): gotControlPlane = %v; wantControlPlane = %v", tc.name, len(tc.inputMachines), actualControlPlane != nil, tc.expectedControlPlane != nil) + if (tc.expectedCPMachines == nil && actualCPMachines != nil) || + (tc.expectedCPMachines != nil && actualCPMachines == nil) { + t.Fatalf("%s: extractControlPlaneMachine(%q): gotControlPlane = %v; wantControlPlane = %v", tc.name, len(tc.inputMachines), actualCPMachines[0] != nil, tc.expectedCPMachines != nil) } - if len(tc.expectedNodes) != len(actualNodes) { - t.Fatalf("%s: extractControlPlaneMachine(%q): gotNodes = %q; wantNodes = %q", tc.name, len(tc.inputMachines), len(actualNodes), len(tc.expectedNodes)) + if len(tc.expectedNodeMachines) != len(actualNodes) { + t.Fatalf("%s: extractControlPlaneMachine(%q): gotNodes = %q; wantNodes = %q", tc.name, len(tc.inputMachines), len(actualNodes), len(tc.expectedNodeMachines)) } }) } @@ -1434,14 +1430,18 @@ func TestClusterDelete(t *testing.T) { } } -func generateTestControlPlaneMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine { - machine := generateTestNodeMachine(cluster, ns, name) - machine.Spec = clusterv1.MachineSpec{ - Versions: clusterv1.MachineVersionInfo{ - ControlPlane: "1.10.1", - }, +func generateTestControlPlaneMachines(cluster *clusterv1.Cluster, ns string, names []string) []*clusterv1.Machine { + machines := make([]*clusterv1.Machine, 0, len(names)) + for _, name := range names { + machine := generateTestNodeMachine(cluster, ns, name) + machine.Spec = clusterv1.MachineSpec{ + Versions: clusterv1.MachineVersionInfo{ + ControlPlane: "1.10.1", + }, + } + machines = append(machines, machine) } - return machine + return machines } func generateTestNodeMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine { @@ -1473,18 +1473,9 @@ func generateTestNodeMachines(cluster *clusterv1.Cluster, ns string, nodeNames [ return nodes } -func generateInvalidExtractControlPlaneMachine(cluster *clusterv1.Cluster, ns string, controlPlaneNames, nodeNames []string) []*clusterv1.Machine { - var machines []*clusterv1.Machine // nolint - for _, name := range controlPlaneNames { - machines = append(machines, generateTestControlPlaneMachine(cluster, ns, name)) - } - machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...) - return machines -} - -func generateValidExtractControlPlaneMachineInput(cluster *clusterv1.Cluster, ns, controlPlaneName string, nodeNames []string) []*clusterv1.Machine { +func generateValidExtractControlPlaneMachineInput(cluster *clusterv1.Cluster, ns string, controlPlaneName []string, nodeNames []string) []*clusterv1.Machine { var machines []*clusterv1.Machine - machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName)) + machines = append(machines, generateTestControlPlaneMachines(cluster, ns, controlPlaneName)...) machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...) return machines } @@ -1497,7 +1488,7 @@ func generateMachines(cluster *clusterv1.Cluster, ns string) []*clusterv1.Machin controlPlaneName = cluster.Name + controlPlaneName workerName = cluster.Name + workerName } - machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName)) + machines = append(machines, generateTestControlPlaneMachines(cluster, ns, []string{controlPlaneName})...) machines = append(machines, generateTestNodeMachine(cluster, ns, workerName)) return machines }