✨clusterctl upgrade apply: wait for providers to be ready

cmd/clusterctl/client/cluster/installer.go

@@ -96,7 +96,7 @@ func (i *providerInstaller) Install(opts InstallOptions) ([]repository.Component
 		ret = append(ret, components)
 	}
 
-	return ret, i.waitForProvidersReady(opts)
+	return ret, waitForProvidersReady(opts, i.installQueue, i.proxy)
 }
 
 func installComponentsAndUpdateInventory(components repository.Components, providerComponents ComponentsClient, providerInventory InventoryClient) error {
@@ -115,7 +115,7 @@ func installComponentsAndUpdateInventory(components repository.Components, provi
 }
 
 // waitForProvidersReady waits till the installed components are ready.
-func (i *providerInstaller) waitForProvidersReady(opts InstallOptions) error {
+func waitForProvidersReady(opts InstallOptions, installQueue []repository.Components, proxy Proxy) error {
 	// If we dont have to wait for providers to be installed
 	// return early.
 	if !opts.WaitProviders {
@@ -125,15 +125,15 @@ func (i *providerInstaller) waitForProvidersReady(opts InstallOptions) error {
 	log := logf.Log
 	log.Info("Waiting for providers to be available...")
 
-	return i.waitManagerDeploymentsReady(opts)
+	return waitManagerDeploymentsReady(opts, installQueue, proxy)
 }
 
 // waitManagerDeploymentsReady waits till the installed manager deployments are ready.
-func (i *providerInstaller) waitManagerDeploymentsReady(opts InstallOptions) error {
-	for _, components := range i.installQueue {
+func waitManagerDeploymentsReady(opts InstallOptions, installQueue []repository.Components, proxy Proxy) error {
+	for _, components := range installQueue {
 		for _, obj := range components.Objs() {
 			if util.IsDeploymentWithManager(obj) {
-				if err := i.waitDeploymentReady(obj, opts.WaitProviderTimeout); err != nil {
+				if err := waitDeploymentReady(obj, opts.WaitProviderTimeout, proxy); err != nil {
 					return err
 				}
 			}
@@ -142,9 +142,9 @@ func (i *providerInstaller) waitManagerDeploymentsReady(opts InstallOptions) err
 	return nil
 }
 
-func (i *providerInstaller) waitDeploymentReady(deployment unstructured.Unstructured, timeout time.Duration) error {
+func waitDeploymentReady(deployment unstructured.Unstructured, timeout time.Duration, proxy Proxy) error {
 	return wait.Poll(100*time.Millisecond, timeout, func() (bool, error) {
-		c, err := i.proxy.NewClient()
+		c, err := proxy.NewClient()
 		if err != nil {
 			return false, err
 		}

cmd/clusterctl/client/cluster/upgrader.go

@@ -44,10 +44,10 @@ type ProviderUpgrader interface {
 	Plan() ([]UpgradePlan, error)
 
 	// ApplyPlan executes an upgrade following an UpgradePlan generated by clusterctl.
-	ApplyPlan(clusterAPIVersion string) error
+	ApplyPlan(opts UpgradeOptions, clusterAPIVersion string) error
 
 	// ApplyCustomPlan plan executes an upgrade using the UpgradeItems provided by the user.
-	ApplyCustomPlan(providersToUpgrade ...UpgradeItem) error
+	ApplyCustomPlan(opts UpgradeOptions, providersToUpgrade ...UpgradeItem) error
 }
 
 // UpgradePlan defines a list of possible upgrade targets for a management cluster.
@@ -56,6 +56,12 @@ type UpgradePlan struct {
 	Providers []UpgradeItem
 }
 
+// UpgradeOptions defines the options used to upgrade installation.
+type UpgradeOptions struct {
+	WaitProviders       bool
+	WaitProviderTimeout time.Duration
+}
+
 // isPartialUpgrade returns true if at least one upgradeItem in the plan does not have a target version.
 func (u *UpgradePlan) isPartialUpgrade() bool {
 	for _, i := range u.Providers {
@@ -149,7 +155,7 @@ func (u *providerUpgrader) Plan() ([]UpgradePlan, error) {
 	return ret, nil
 }
 
-func (u *providerUpgrader) ApplyPlan(contract string) error {
+func (u *providerUpgrader) ApplyPlan(opts UpgradeOptions, contract string) error {
 	if contract != clusterv1.GroupVersion.Version {
 		return errors.Errorf("current version of clusterctl could only upgrade to %s contract, requested %s", clusterv1.GroupVersion.Version, contract)
 	}
@@ -169,10 +175,10 @@ func (u *providerUpgrader) ApplyPlan(contract string) error {
 	}
 
 	// Do the upgrade
-	return u.doUpgrade(upgradePlan)
+	return u.doUpgrade(upgradePlan, opts)
 }
 
-func (u *providerUpgrader) ApplyCustomPlan(upgradeItems ...UpgradeItem) error {
+func (u *providerUpgrader) ApplyCustomPlan(opts UpgradeOptions, upgradeItems ...UpgradeItem) error {
 	log := logf.Log
 	log.Info("Performing upgrade...")
 
@@ -184,7 +190,7 @@ func (u *providerUpgrader) ApplyCustomPlan(upgradeItems ...UpgradeItem) error {
 	}
 
 	// Do the upgrade
-	return u.doUpgrade(upgradePlan)
+	return u.doUpgrade(upgradePlan, opts)
 }
 
 // getUpgradePlan returns the upgrade plan for a specific set of providers/contract
@@ -346,7 +352,7 @@ func (u *providerUpgrader) getUpgradeComponents(provider UpgradeItem) (repositor
 	return components, nil
 }
 
-func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error {
+func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan, opts UpgradeOptions) error {
 	// Check for multiple instances of the same provider if current contract is v1alpha3.
 	if upgradePlan.Contract == clusterv1.GroupVersion.Version {
 		if err := u.providerInventory.CheckSingleProviderInstance(); err != nil {
@@ -380,6 +386,8 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error {
 		}
 	}
 
+	installQueue := []repository.Components{}
+
 	// Delete old providers and deploy new ones if necessary, i.e. there is a NextVersion.
 	for _, upgradeItem := range providers {
 		// If there is not a specified next version, skip it (we are already up-to-date).
@@ -393,6 +401,8 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error {
 			return err
 		}
 
+		installQueue = append(installQueue, components)
+
 		// Delete the provider, preserving CRD, namespace and the inventory.
 		if err := u.providerComponents.Delete(DeleteOptions{
 			Provider:         upgradeItem.Provider,
@@ -416,7 +426,7 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error {
 		}
 	}
 
-	return nil
+	return waitForProvidersReady(InstallOptions(opts), installQueue, u.proxy)
 }
 
 func (u *providerUpgrader) scaleDownProvider(provider clusterctlv1.Provider) error {

cmd/clusterctl/client/cluster/upgrader_test.go

@@ -888,6 +888,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) {
 		contract string
 		wantErr  bool
 		errorMsg string
+		opts     UpgradeOptions
 	}{
 		{
 			name: "fails to upgrade to current contract when there are multiple instances of the core provider",
@@ -924,6 +925,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) {
 			contract: test.CurrentCAPIContract,
 			wantErr:  true,
 			errorMsg: "detected multiple instances of the same provider",
+			opts:     UpgradeOptions{},
 		},
 		{
 			name: "fails to upgrade to current contract when there are multiple instances of the infra provider",
@@ -960,6 +962,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) {
 			contract: test.CurrentCAPIContract,
 			wantErr:  true,
 			errorMsg: "detected multiple instances of the same provider",
+			opts:     UpgradeOptions{},
 		},
 	}
 
@@ -976,7 +979,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) {
 				},
 				providerInventory: newInventoryClient(tt.fields.proxy, nil),
 			}
-			err := u.ApplyPlan(tt.contract)
+			err := u.ApplyPlan(tt.opts, tt.contract)
 			if tt.wantErr {
 				g.Expect(err).To(HaveOccurred())
 				g.Expect(err.Error()).Should(ContainSubstring(tt.errorMsg))
@@ -1002,6 +1005,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) {
 		providersToUpgrade []UpgradeItem
 		wantErr            bool
 		errorMsg           string
+		opts               UpgradeOptions
 	}{
 		{
 			name: "fails to upgrade to v1alpha4 when there are multiple instances of the core provider",
@@ -1047,6 +1051,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) {
 			},
 			wantErr:  true,
 			errorMsg: "invalid management cluster: there should a core provider, found 2",
+			opts:     UpgradeOptions{},
 		},
 		{
 			name: "fails to upgrade to v1alpha4 when there are multiple instances of the infra provider",
@@ -1096,6 +1101,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) {
 			},
 			wantErr:  true,
 			errorMsg: "detected multiple instances of the same provider",
+			opts:     UpgradeOptions{},
 		},
 	}
 
@@ -1112,7 +1118,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) {
 				},
 				providerInventory: newInventoryClient(tt.fields.proxy, nil),
 			}
-			err := u.ApplyCustomPlan(tt.providersToUpgrade...)
+			err := u.ApplyCustomPlan(tt.opts, tt.providersToUpgrade...)
 			if tt.wantErr {
 				g.Expect(err).To(HaveOccurred())
 				g.Expect(err.Error()).Should(ContainSubstring(tt.errorMsg))

cmd/clusterctl/client/upgrade.go

@@ -18,6 +18,7 @@ package client
 
 import (
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -107,6 +108,12 @@ type ApplyUpgradeOptions struct {
 
 	// InfrastructureProviders instance and versions (e.g. capa-system/aws:v0.5.0) to upgrade to. This field can be used as alternative to Contract.
 	InfrastructureProviders []string
+
+	// WaitProviders instructs the upgrade apply command to wait till the providers are successfully upgraded.
+	WaitProviders bool
+
+	// WaitProviderTimeout sets the timeout per provider upgrade.
+	WaitProviderTimeout time.Duration
 }
 
 func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
@@ -150,6 +157,11 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
 		len(options.ControlPlaneProviders) > 0 ||
 		len(options.InfrastructureProviders) > 0
 
+	opts := cluster.UpgradeOptions{
+		WaitProviders:       options.WaitProviders,
+		WaitProviderTimeout: options.WaitProviderTimeout,
+	}
+
 	// If we are upgrading a specific set of providers only, process the providers and call ApplyCustomPlan.
 	if isCustomUpgrade {
 		// Converts upgrade references back into an UpgradeItem.
@@ -175,11 +187,11 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
 		}
 
 		// Execute the upgrade using the custom upgrade items
-		return clusterClient.ProviderUpgrader().ApplyCustomPlan(upgradeItems...)
+		return clusterClient.ProviderUpgrader().ApplyCustomPlan(opts, upgradeItems...)
 	}
 
 	// Otherwise we are upgrading a whole management cluster according to a clusterctl generated upgrade plan.
-	return clusterClient.ProviderUpgrader().ApplyPlan(options.Contract)
+	return clusterClient.ProviderUpgrader().ApplyPlan(opts, options.Contract)
 }
 
 func addUpgradeItems(upgradeItems []cluster.UpgradeItem, providerType clusterctlv1.ProviderType, providers ...string) ([]cluster.UpgradeItem, error) {

cmd/clusterctl/cmd/upgrade_apply.go

@@ -17,6 +17,8 @@ limitations under the License.
 package cmd
 
 import (
+	"time"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
@@ -31,6 +33,8 @@ type upgradeApplyOptions struct {
 	bootstrapProviders      []string
 	controlPlaneProviders   []string
 	infrastructureProviders []string
+	waitProviders           bool
+	waitProviderTimeout     int
 }
 
 var ua = &upgradeApplyOptions{}
@@ -73,6 +77,10 @@ func init() {
 		"Bootstrap providers instance and versions (e.g. capi-kubeadm-bootstrap-system/kubeadm:v0.3.0) to upgrade to. This flag can be used as alternative to --contract.")
 	upgradeApplyCmd.Flags().StringSliceVarP(&ua.controlPlaneProviders, "control-plane", "c", nil,
 		"ControlPlane providers instance and versions (e.g. capi-kubeadm-control-plane-system/kubeadm:v0.3.0) to upgrade to. This flag can be used as alternative to --contract.")
+	upgradeApplyCmd.Flags().BoolVar(&ua.waitProviders, "wait-providers", false,
+		"Wait for providers to be upgraded.")
+	upgradeApplyCmd.Flags().IntVar(&ua.waitProviderTimeout, "wait-provider-timeout", 5*60,
+		"Wait timeout per provider upgrade in seconds. This value is ignored if --wait-providers is false")
 }
 
 func runUpgradeApply() error {
@@ -100,5 +108,7 @@ func runUpgradeApply() error {
 		BootstrapProviders:      ua.bootstrapProviders,
 		ControlPlaneProviders:   ua.controlPlaneProviders,
 		InfrastructureProviders: ua.infrastructureProviders,
+		WaitProviders:           ua.waitProviders,
+		WaitProviderTimeout:     time.Duration(ua.waitProviderTimeout) * time.Second,
 	})
 }