Skip to content

Commit

Permalink
address feedbacks
Browse files Browse the repository at this point in the history
  • Loading branch information
@fabriziopandini
fabriziopandini committed Apr 28, 2021
1 parent 706caea commit 8b19078
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 61 deletions.
52 changes: 27 additions & 25 deletions test/e2e/clusterctl_upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const initWithBinaryVariableName = "INIT_WITH_BINARY"

// ClusterctlUpgradeSpecInput is the input for ClusterctlUpgradeSpec.
type ClusterctlUpgradeSpecInput struct {
E2EConfig *clusterctl.E2EConfig
Expand Down Expand Up @@ -76,8 +78,10 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
Expect(input.E2EConfig.Variables).To(HaveKey(initWithBinaryVariableName), "Invalid argument. INIT_WITH_BINARY variable must be defined when calling %s spec", specName)
Expect(input.E2EConfig.Variables[initWithBinaryVariableName]).ToNot(BeEmpty(), "Invalid argument. INIT_WITH_BINARY variable can't be empty when calling %s spec", specName)
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
managementClusterNamespace, managementClusterCancelWatches = setupSpecNamespace(context.TODO(), specName, input.BootstrapClusterProxy, input.ArtifactFolder)
Expand All @@ -89,7 +93,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// NOTE: given that the bootstrap cluster could be shared by several tests, it is not practical to use it for testing clusterctl upgrades.
// So we are creating a workload cluster that will be used as a new management cluster where to install older version of providers

clusterctl.ApplyClusterTemplateAndWait(context.TODO(), clusterctl.ApplyClusterTemplateAndWaitInput{
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: input.BootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
Expand Down Expand Up @@ -117,32 +121,30 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// built locally and loaded into kind
cluster := managementClusterResources.Cluster
if cluster.Spec.InfrastructureRef.Kind == "DockerCluster" {
Expect(bootstrap.LoadImagesToKindCluster(context.TODO(), bootstrap.LoadImagesToKindClusterInput{
Expect(bootstrap.LoadImagesToKindCluster(ctx, bootstrap.LoadImagesToKindClusterInput{
Name: cluster.Name,
Images: input.E2EConfig.Images,
})).To(Succeed())
}

// Get a ClusterBroker so we can interact with the workload cluster
managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(context.TODO(), cluster.Namespace, cluster.Name)
managementClusterProxy = input.BootstrapClusterProxy.GetWorkloadCluster(ctx, cluster.Namespace, cluster.Name)

var binaryURL string
if input.E2EConfig.HasVariable("INIT_WITH_BINARY") {
binaryURL = input.E2EConfig.GetVariable("INIT_WITH_BINARY")
binaryURL = strings.ReplaceAll(binaryURL, "{OS}", runtime.GOOS)
binaryURL = strings.ReplaceAll(binaryURL, "{ARCH}", runtime.GOARCH)
}
// Download the v1alpha3 clusterctl version to be used for setting up the management cluster to be upgraded
clusterctlBinaryURL := input.E2EConfig.GetVariable(initWithBinaryVariableName)
clusterctlBinaryURL = strings.ReplaceAll(clusterctlBinaryURL, "{OS}", runtime.GOOS)
clusterctlBinaryURL = strings.ReplaceAll(clusterctlBinaryURL, "{ARCH}", runtime.GOARCH)

log.Logf("downloading clusterctl binary from %s", binaryURL)
binaryFile := downloadToTmpFile(binaryURL)
defer os.Remove(binaryFile) // clean up
log.Logf("downloading clusterctl binary from %s", clusterctlBinaryURL)
binaryFileclusterctlBinaryPath := downloadToTmpFile(clusterctlBinaryURL)
defer os.Remove(binaryFileclusterctlBinaryPath) // clean up

err := os.Chmod(binaryFile, 0744)
err := os.Chmod(binaryFileclusterctlBinaryPath, 0744)
Expect(err).ToNot(HaveOccurred(), "failed to chmod temporary file")

By("Initializing the workload cluster with older versions of providers")
clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{
BinaryFile: binaryFile, // use older version of clusterctl to init the management cluster
clusterctl.InitManagementClusterAndWatchControllerLogs(ctx, clusterctl.InitManagementClusterAndWatchControllerLogsInput{
ClusterctlBinaryPath: binaryFileclusterctlBinaryPath, // use older version of clusterctl to init the management cluster
ClusterProxy: managementClusterProxy,
ClusterctlConfigPath: input.ClusterctlConfigPath,
CoreProvider: input.E2EConfig.GetProvidersWithOldestVersion(config.ClusterAPIProviderName)[0],
Expand All @@ -155,7 +157,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
By("THE MANAGEMENT CLUSTER WITH THE OLDER VERSION OF PROVIDERS IS UP&RUNNING!")

Byf("Creating a namespace for hosting the %s test workload cluster", specName)
testNamespace, testCancelWatches = framework.CreateNamespaceAndWatchEvents(context.TODO(), framework.CreateNamespaceAndWatchEventsInput{
testNamespace, testCancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
Creator: managementClusterProxy.GetClient(),
ClientSet: managementClusterProxy.GetClientSet(),
Name: specName,
Expand All @@ -177,7 +179,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
clusterName, "(default)", kubernetesVersion, controlPlaneMachineCount, workerMachineCount)

log.Logf("Getting the cluster template yaml")
workloadClusterTemplate := clusterctl.ConfigClusterWithBinary(ctx, binaryFile, clusterctl.ConfigClusterInput{
workloadClusterTemplate := clusterctl.ConfigClusterWithBinary(ctx, binaryFileclusterctlBinaryPath, clusterctl.ConfigClusterInput{
// pass reference to the management cluster hosting this test
KubeconfigPath: managementClusterProxy.GetKubeconfigPath(),
// pass the clusterctl config file that points to the local provider repository created for this test,
Expand Down Expand Up @@ -216,7 +218,7 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
By("THE MANAGEMENT CLUSTER WITH OLDER VERSION OF PROVIDERS WORKS!")

By("Upgrading providers to the latest version available")
clusterctl.UpgradeManagementClusterAndWait(context.TODO(), clusterctl.UpgradeManagementClusterAndWaitInput{
clusterctl.UpgradeManagementClusterAndWait(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{
ClusterctlConfigPath: input.ClusterctlConfigPath,
ClusterProxy: managementClusterProxy,
ManagementGroup: fmt.Sprintf("capi-system/%s", config.ClusterAPIProviderName),
Expand All @@ -229,13 +231,13 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
// After upgrading we are sure the version is the latest version of the API,
// so it is possible to use the standard helpers

testMachineDeployments := framework.GetMachineDeploymentsByCluster(context.TODO(), framework.GetMachineDeploymentsByClusterInput{
testMachineDeployments := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{
Lister: managementClusterProxy.GetClient(),
ClusterName: clusterName,
Namespace: testNamespace.Name,
})

framework.ScaleAndWaitMachineDeployment(context.TODO(), framework.ScaleAndWaitMachineDeploymentInput{
framework.ScaleAndWaitMachineDeployment(ctx, framework.ScaleAndWaitMachineDeploymentInput{
ClusterProxy: managementClusterProxy,
Cluster: &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace.Name}},
MachineDeployment: testMachineDeployments[0],
Expand Down Expand Up @@ -281,18 +283,18 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
}

func downloadToTmpFile(url string) string {
tmpfile, err := ioutil.TempFile("", "clusterctl")
tmpFile, err := ioutil.TempFile("", "clusterctl")
Expect(err).ToNot(HaveOccurred(), "failed to get temporary file")
defer tmpfile.Close()
defer tmpFile.Close()

// Get the data
resp, err := http.Get(url)
Expect(err).ToNot(HaveOccurred(), "failed to get clusterctl")
defer resp.Body.Close()

// Write the body to file
_, err = io.Copy(tmpfile, resp.Body)
_, err = io.Copy(tmpFile, resp.Body)
Expect(err).ToNot(HaveOccurred(), "failed to write temporary file")

return tmpfile.Name()
return tmpFile.Name()
}
16 changes: 8 additions & 8 deletions test/framework/clusterctl/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ type InitInput struct {
func Init(ctx context.Context, input InitInput) {
log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s",
input.CoreProvider,
strings.Join(input.BootstrapProviders, ", "),
strings.Join(input.ControlPlaneProviders, ", "),
strings.Join(input.InfrastructureProviders, ", "),
strings.Join(input.BootstrapProviders, ","),
strings.Join(input.ControlPlaneProviders, ","),
strings.Join(input.InfrastructureProviders, ","),
)

initOpt := clusterctlclient.InitOptions{
Expand All @@ -89,9 +89,9 @@ func Init(ctx context.Context, input InitInput) {
func InitWithBinary(_ context.Context, binary string, input InitInput) {
log.Logf("clusterctl init --core %s --bootstrap %s --control-plane %s --infrastructure %s",
input.CoreProvider,
strings.Join(input.BootstrapProviders, ", "),
strings.Join(input.ControlPlaneProviders, ", "),
strings.Join(input.InfrastructureProviders, ", "),
strings.Join(input.BootstrapProviders, ","),
strings.Join(input.ControlPlaneProviders, ","),
strings.Join(input.InfrastructureProviders, ","),
)

cmd := exec.Command(binary, "init",
Expand Down Expand Up @@ -196,7 +196,7 @@ func ConfigCluster(ctx context.Context, input ConfigClusterInput) []byte {
}

// ConfigClusterWithBinary uses clusterctl binary to run config cluster.
func ConfigClusterWithBinary(_ context.Context, binary string, input ConfigClusterInput) []byte {
func ConfigClusterWithBinary(_ context.Context, clusterctlBinaryPath string, input ConfigClusterInput) []byte {
log.Logf("clusterctl config cluster %s --infrastructure %s --kubernetes-version %s --control-plane-machine-count %d --worker-machine-count %d --flavor %s",
input.ClusterName,
valueOrDefault(input.InfrastructureProvider),
Expand All @@ -206,7 +206,7 @@ func ConfigClusterWithBinary(_ context.Context, binary string, input ConfigClust
valueOrDefault(input.Flavor),
)

cmd := exec.Command(binary, "config", "cluster",
cmd := exec.Command(clusterctlBinaryPath, "config", "cluster",
input.ClusterName,
"--infrastructure", input.InfrastructureProvider,
"--kubernetes-version", input.InfrastructureProvider,
Expand Down
46 changes: 18 additions & 28 deletions test/framework/clusterctl/clusterctl_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type InitManagementClusterAndWatchControllerLogsInput struct {
InfrastructureProviders []string
LogFolder string
DisableMetricsCollection bool
BinaryFile string
ClusterctlBinaryPath string
}

// InitManagementClusterAndWatchControllerLogs initializes a management using clusterctl and setup watches for controller logs.
Expand Down Expand Up @@ -69,34 +69,24 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init
Lister: client,
})
if len(controllersDeployments) == 0 {
if input.BinaryFile != "" {
InitWithBinary(ctx, input.BinaryFile, InitInput{
// pass reference to the management cluster hosting this test
KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(),
// pass the clusterctl config file that points to the local provider repository created for this test
ClusterctlConfigPath: input.ClusterctlConfigPath,
// setup the desired list of providers for a single-tenant management cluster
CoreProvider: input.CoreProvider,
BootstrapProviders: input.BootstrapProviders,
ControlPlaneProviders: input.ControlPlaneProviders,
InfrastructureProviders: input.InfrastructureProviders,
// setup clusterctl logs folder
LogFolder: input.LogFolder,
})
initInput := InitInput{
// pass reference to the management cluster hosting this test
KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(),
// pass the clusterctl config file that points to the local provider repository created for this test
ClusterctlConfigPath: input.ClusterctlConfigPath,
// setup the desired list of providers for a single-tenant management cluster
CoreProvider: input.CoreProvider,
BootstrapProviders: input.BootstrapProviders,
ControlPlaneProviders: input.ControlPlaneProviders,
InfrastructureProviders: input.InfrastructureProviders,
// setup clusterctl logs folder
LogFolder: input.LogFolder,
}

if input.ClusterctlBinaryPath != "" {
InitWithBinary(ctx, input.ClusterctlBinaryPath, initInput)
} else {
Init(ctx, InitInput{
// pass reference to the management cluster hosting this test
KubeconfigPath: input.ClusterProxy.GetKubeconfigPath(),
// pass the clusterctl config file that points to the local provider repository created for this test
ClusterctlConfigPath: input.ClusterctlConfigPath,
// setup the desired list of providers for a single-tenant management cluster
CoreProvider: input.CoreProvider,
BootstrapProviders: input.BootstrapProviders,
ControlPlaneProviders: input.ControlPlaneProviders,
InfrastructureProviders: input.InfrastructureProviders,
// setup clusterctl logs folder
LogFolder: input.LogFolder,
})
Init(ctx, initInput)
}
}

Expand Down

0 comments on commit 8b19078

Please sign in to comment.