Skip to content

Commit

Permalink
Ability to deploy multiple control plane machines (#878)
Browse files Browse the repository at this point in the history 
* Ability to deploy multiple control plane machines

* Update test

* Done serially until kubernetes/kubeadm#1097 is resolved

* Remove dishonest test case
  • Loading branch information
@h0tbird @k8s-ci-robot
h0tbird authored and k8s-ci-robot committed Apr 5, 2019
1 parent b90a4cc commit 767e4ba
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 64 deletions.
15 changes: 7 additions & 8 deletions cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1019,17 +1019,16 @@ func GetClusterAPIObject(client Client, clusterName, namespace string) (*cluster
return nil, nil, nil, errors.Wrapf(err, "unable to fetch cluster %s/%s", namespace, clusterName)
}

controlPlane, nodes, err := ExtractControlPlaneMachine(machines)
controlPlane, nodes, err := ExtractControlPlaneMachines(machines)
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "unable to fetch control plane machine in cluster %s/%s", namespace, clusterName)
}
return cluster, controlPlane, nodes, nil
return cluster, controlPlane[0], nodes, nil
}

// ExtractControlPlaneMachine separates the machines running the control plane (singular) from the incoming machines.
// ExtractControlPlaneMachines separates the machines running the control plane from the incoming machines.
// This is currently done by looking at which machine specifies the control plane version.
// TODO: Cleanup.
func ExtractControlPlaneMachine(machines []*clusterv1.Machine) (*clusterv1.Machine, []*clusterv1.Machine, error) {
func ExtractControlPlaneMachines(machines []*clusterv1.Machine) ([]*clusterv1.Machine, []*clusterv1.Machine, error) {
nodes := []*clusterv1.Machine{}
controlPlaneMachines := []*clusterv1.Machine{}
for _, machine := range machines {
Expand All @@ -1039,8 +1038,8 @@ func ExtractControlPlaneMachine(machines []*clusterv1.Machine) (*clusterv1.Machi
nodes = append(nodes, machine)
}
}
if len(controlPlaneMachines) != 1 {
return nil, nil, errors.Errorf("expected one control plane machine, got: %v", len(controlPlaneMachines))
if len(controlPlaneMachines) < 1 {
return nil, nil, errors.Errorf("expected one or more control plane machines, got: %v", len(controlPlaneMachines))
}
return controlPlaneMachines[0], nodes, nil
return controlPlaneMachines, nodes, nil
}
21 changes: 16 additions & 5 deletions cmd/clusterctl/clusterdeployer/clusterdeployer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func New(

// Create the cluster from the provided cluster definition and machine list.
func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*clusterv1.Machine, provider provider.Deployer, kubeconfigOutput string, providerComponentsStoreFactory provider.ComponentsStoreFactory) error {
controlPlaneMachine, nodes, err := clusterclient.ExtractControlPlaneMachine(machines)
controlPlaneMachines, nodes, err := clusterclient.ExtractControlPlaneMachines(machines)
if err != nil {
return errors.Wrap(err, "unable to separate control plane machines from node machines")
}
Expand Down Expand Up @@ -89,12 +89,12 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster
cluster.Namespace = bootstrapClient.GetContextNamespace()
}

klog.Infof("Creating control plane %v in namespace %q", controlPlaneMachine.Name, cluster.Namespace)
if err := phases.ApplyMachines(bootstrapClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachine}); err != nil {
klog.Infof("Creating control plane %v in namespace %q", controlPlaneMachines[0].Name, cluster.Namespace)
if err := phases.ApplyMachines(bootstrapClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachines[0]}); err != nil {
return errors.Wrap(err, "unable to create control plane machine")
}

klog.Infof("Updating bootstrap cluster object for cluster %v in namespace %q with control plane endpoint running on %s", cluster.Name, cluster.Namespace, controlPlaneMachine.Name)
klog.Infof("Updating bootstrap cluster object for cluster %v in namespace %q with control plane endpoint running on %s", cluster.Name, cluster.Namespace, controlPlaneMachines[0].Name)
if err := d.updateClusterEndpoint(bootstrapClient, provider, cluster.Name, cluster.Namespace); err != nil {
return errors.Wrap(err, "unable to update bootstrap cluster endpoint")
}
Expand Down Expand Up @@ -130,11 +130,22 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster

// For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we
// update the target cluster endpoint as well to be sure.
klog.Infof("Updating target cluster object with control plane endpoint running on %s", controlPlaneMachine.Name)
klog.Infof("Updating target cluster object with control plane endpoint running on %s", controlPlaneMachines[0].Name)
if err := d.updateClusterEndpoint(targetClient, provider, cluster.Name, cluster.Namespace); err != nil {
return errors.Wrap(err, "unable to update target cluster endpoint")
}

if len(controlPlaneMachines) > 1 {
// TODO(h0tbird) Done serially until kubernetes/kubeadm#1097 is resolved and all
// supported versions of k8s we are deploying (using kubeadm) have the fix.
klog.Info("Creating additional control plane machines in target cluster.")
for _, controlPlaneMachine := range controlPlaneMachines[1:] {
if err := phases.ApplyMachines(targetClient, cluster.Namespace, []*clusterv1.Machine{controlPlaneMachine}); err != nil {
return errors.Wrap(err, "unable to create additional control plane machines")
}
}
}

klog.Info("Creating node machines in target cluster.")
if err := phases.ApplyMachines(targetClient, cluster.Namespace, nodes); err != nil {
return errors.Wrap(err, "unable to create node machines")
Expand Down
93 changes: 42 additions & 51 deletions cmd/clusterctl/clusterdeployer/clusterdeployer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,17 +660,6 @@ func TestClusterCreate(t *testing.T) {
expectExternalExists: false,
expectExternalCreated: true,
},
{
name: "fail provision multiple clusters in a namespace",
targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }},
bootstrapClient: &testClusterClient{},
namespaceToExpectedInternalMachines: make(map[string]int),
namespaceToInputCluster: map[string][]*clusterv1.Cluster{"foo": getClustersForNamespace("foo", 3)},
expectErr: true,
cleanupExternal: true,
expectExternalExists: false,
expectExternalCreated: true,
},
{
name: "fail provision bootstrap cluster",
targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }},
Expand Down Expand Up @@ -996,42 +985,49 @@ func TestExtractControlPlaneMachine(t *testing.T) {
testCases := []struct {
name string
inputMachines []*clusterv1.Machine
expectedControlPlane *clusterv1.Machine
expectedNodes []*clusterv1.Machine
expectedCPMachines []*clusterv1.Machine
expectedNodeMachines []*clusterv1.Machine
expectedError error
}{
{
name: "success_1_control_plane_1_node",
inputMachines: generateMachines(nil, metav1.NamespaceDefault),
expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName),
expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}),
expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}),
expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}),
expectedError: nil,
},
{
name: "success_1_control_plane_multiple_nodes",
inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, singleControlPlaneName, multipleNodeNames),
expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName),
expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames),
inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}, multipleNodeNames),
expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, []string{singleControlPlaneName}),
expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames),
expectedError: nil,
},
{
name: "fail_more_than_1_control_plane_not_allowed",
inputMachines: generateInvalidExtractControlPlaneMachine(nil, metav1.NamespaceDefault, multipleControlPlaneNames, multipleNodeNames),
expectedControlPlane: nil,
expectedNodes: nil,
expectedError: errors.New("expected one control plane machine, got: 2"),
name: "success_2_control_planes_1_node",
inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, multipleControlPlaneNames, []string{singleNodeName}),
expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, multipleControlPlaneNames),
expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}),
expectedError: nil,
},
{
name: "success_2_control_planes_multiple_nodes",
inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, multipleControlPlaneNames, multipleNodeNames),
expectedCPMachines: generateTestControlPlaneMachines(nil, metav1.NamespaceDefault, multipleControlPlaneNames),
expectedNodeMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames),
expectedError: nil,
},
{
name: "fail_0_control_plane_not_allowed",
inputMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames),
expectedControlPlane: nil,
expectedNodes: nil,
expectedError: errors.New("expected one control plane machine, got: 0"),
expectedCPMachines: nil,
expectedNodeMachines: nil,
expectedError: errors.New("expected one or more control plane machines, got: 0"),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
actualControlPlane, actualNodes, actualError := clusterclient.ExtractControlPlaneMachine(tc.inputMachines)
actualCPMachines, actualNodes, actualError := clusterclient.ExtractControlPlaneMachines(tc.inputMachines)

if tc.expectedError == nil && actualError != nil {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotError %q; wantError [nil]", tc.name, len(tc.inputMachines), actualError)
Expand All @@ -1041,13 +1037,13 @@ func TestExtractControlPlaneMachine(t *testing.T) {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotError %q; wantError %q", tc.name, len(tc.inputMachines), actualError, tc.expectedError)
}

if (tc.expectedControlPlane == nil && actualControlPlane != nil) ||
(tc.expectedControlPlane != nil && actualControlPlane == nil) {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotControlPlane = %v; wantControlPlane = %v", tc.name, len(tc.inputMachines), actualControlPlane != nil, tc.expectedControlPlane != nil)
if (tc.expectedCPMachines == nil && actualCPMachines != nil) ||
(tc.expectedCPMachines != nil && actualCPMachines == nil) {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotControlPlane = %v; wantControlPlane = %v", tc.name, len(tc.inputMachines), actualCPMachines[0] != nil, tc.expectedCPMachines != nil)
}

if len(tc.expectedNodes) != len(actualNodes) {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotNodes = %q; wantNodes = %q", tc.name, len(tc.inputMachines), len(actualNodes), len(tc.expectedNodes))
if len(tc.expectedNodeMachines) != len(actualNodes) {
t.Fatalf("%s: extractControlPlaneMachine(%q): gotNodes = %q; wantNodes = %q", tc.name, len(tc.inputMachines), len(actualNodes), len(tc.expectedNodeMachines))
}
})
}
Expand Down Expand Up @@ -1434,14 +1430,18 @@ func TestClusterDelete(t *testing.T) {
}
}

func generateTestControlPlaneMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine {
machine := generateTestNodeMachine(cluster, ns, name)
machine.Spec = clusterv1.MachineSpec{
Versions: clusterv1.MachineVersionInfo{
ControlPlane: "1.10.1",
},
func generateTestControlPlaneMachines(cluster *clusterv1.Cluster, ns string, names []string) []*clusterv1.Machine {
machines := make([]*clusterv1.Machine, 0, len(names))
for _, name := range names {
machine := generateTestNodeMachine(cluster, ns, name)
machine.Spec = clusterv1.MachineSpec{
Versions: clusterv1.MachineVersionInfo{
ControlPlane: "1.10.1",
},
}
machines = append(machines, machine)
}
return machine
return machines
}

func generateTestNodeMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine {
Expand Down Expand Up @@ -1473,18 +1473,9 @@ func generateTestNodeMachines(cluster *clusterv1.Cluster, ns string, nodeNames [
return nodes
}

func generateInvalidExtractControlPlaneMachine(cluster *clusterv1.Cluster, ns string, controlPlaneNames, nodeNames []string) []*clusterv1.Machine {
var machines []*clusterv1.Machine // nolint
for _, name := range controlPlaneNames {
machines = append(machines, generateTestControlPlaneMachine(cluster, ns, name))
}
machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...)
return machines
}

func generateValidExtractControlPlaneMachineInput(cluster *clusterv1.Cluster, ns, controlPlaneName string, nodeNames []string) []*clusterv1.Machine {
func generateValidExtractControlPlaneMachineInput(cluster *clusterv1.Cluster, ns string, controlPlaneName []string, nodeNames []string) []*clusterv1.Machine {
var machines []*clusterv1.Machine
machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName))
machines = append(machines, generateTestControlPlaneMachines(cluster, ns, controlPlaneName)...)
machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...)
return machines
}
Expand All @@ -1497,7 +1488,7 @@ func generateMachines(cluster *clusterv1.Cluster, ns string) []*clusterv1.Machin
controlPlaneName = cluster.Name + controlPlaneName
workerName = cluster.Name + workerName
}
machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName))
machines = append(machines, generateTestControlPlaneMachines(cluster, ns, []string{controlPlaneName})...)
machines = append(machines, generateTestNodeMachine(cluster, ns, workerName))
return machines
}
Expand Down

0 comments on commit 767e4ba

Please sign in to comment.