From 1bb1142e4dd89ca37cd29c1192bcd7992249ec42 Mon Sep 17 00:00:00 2001 From: Praveen Rewar <8457124+praveenrewar@users.noreply.github.com> Date: Sat, 30 Apr 2022 18:13:09 +0530 Subject: [PATCH] add wait-providers flags to clusterctl upgrade apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clusterctl upgrade apply: Add --wait-providers flags that can be used to wait for providers to be updated Co-Authored-By: Stefan Büringer <4662360+sbueringer@users.noreply.github.com> --- cmd/clusterctl/client/cluster/installer.go | 16 ++++++------ cmd/clusterctl/client/cluster/upgrader.go | 26 +++++++++++++------ .../client/cluster/upgrader_test.go | 10 +++++-- cmd/clusterctl/client/upgrade.go | 16 ++++++++++-- cmd/clusterctl/cmd/upgrade_apply.go | 10 +++++++ 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/cmd/clusterctl/client/cluster/installer.go b/cmd/clusterctl/client/cluster/installer.go index ee0e605a7e8f..bedf189081a4 100644 --- a/cmd/clusterctl/client/cluster/installer.go +++ b/cmd/clusterctl/client/cluster/installer.go @@ -96,7 +96,7 @@ func (i *providerInstaller) Install(opts InstallOptions) ([]repository.Component ret = append(ret, components) } - return ret, i.waitForProvidersReady(opts) + return ret, waitForProvidersReady(opts, i.installQueue, i.proxy) } func installComponentsAndUpdateInventory(components repository.Components, providerComponents ComponentsClient, providerInventory InventoryClient) error { @@ -115,7 +115,7 @@ func installComponentsAndUpdateInventory(components repository.Components, provi } // waitForProvidersReady waits till the installed components are ready. -func (i *providerInstaller) waitForProvidersReady(opts InstallOptions) error { +func waitForProvidersReady(opts InstallOptions, installQueue []repository.Components, proxy Proxy) error { // If we dont have to wait for providers to be installed // return early. if !opts.WaitProviders { @@ -125,15 +125,15 @@ func (i *providerInstaller) waitForProvidersReady(opts InstallOptions) error { log := logf.Log log.Info("Waiting for providers to be available...") - return i.waitManagerDeploymentsReady(opts) + return waitManagerDeploymentsReady(opts, installQueue, proxy) } // waitManagerDeploymentsReady waits till the installed manager deployments are ready. -func (i *providerInstaller) waitManagerDeploymentsReady(opts InstallOptions) error { - for _, components := range i.installQueue { +func waitManagerDeploymentsReady(opts InstallOptions, installQueue []repository.Components, proxy Proxy) error { + for _, components := range installQueue { for _, obj := range components.Objs() { if util.IsDeploymentWithManager(obj) { - if err := i.waitDeploymentReady(obj, opts.WaitProviderTimeout); err != nil { + if err := waitDeploymentReady(obj, opts.WaitProviderTimeout, proxy); err != nil { return err } } @@ -142,9 +142,9 @@ func (i *providerInstaller) waitManagerDeploymentsReady(opts InstallOptions) err return nil } -func (i *providerInstaller) waitDeploymentReady(deployment unstructured.Unstructured, timeout time.Duration) error { +func waitDeploymentReady(deployment unstructured.Unstructured, timeout time.Duration, proxy Proxy) error { return wait.Poll(100*time.Millisecond, timeout, func() (bool, error) { - c, err := i.proxy.NewClient() + c, err := proxy.NewClient() if err != nil { return false, err } diff --git a/cmd/clusterctl/client/cluster/upgrader.go b/cmd/clusterctl/client/cluster/upgrader.go index 7e7100328745..629dcb5b172e 100644 --- a/cmd/clusterctl/client/cluster/upgrader.go +++ b/cmd/clusterctl/client/cluster/upgrader.go @@ -44,10 +44,10 @@ type ProviderUpgrader interface { Plan() ([]UpgradePlan, error) // ApplyPlan executes an upgrade following an UpgradePlan generated by clusterctl. - ApplyPlan(clusterAPIVersion string) error + ApplyPlan(opts UpgradeOptions, clusterAPIVersion string) error // ApplyCustomPlan plan executes an upgrade using the UpgradeItems provided by the user. - ApplyCustomPlan(providersToUpgrade ...UpgradeItem) error + ApplyCustomPlan(opts UpgradeOptions, providersToUpgrade ...UpgradeItem) error } // UpgradePlan defines a list of possible upgrade targets for a management cluster. @@ -56,6 +56,12 @@ type UpgradePlan struct { Providers []UpgradeItem } +// UpgradeOptions defines the options used to upgrade installation. +type UpgradeOptions struct { + WaitProviders bool + WaitProviderTimeout time.Duration +} + // isPartialUpgrade returns true if at least one upgradeItem in the plan does not have a target version. func (u *UpgradePlan) isPartialUpgrade() bool { for _, i := range u.Providers { @@ -149,7 +155,7 @@ func (u *providerUpgrader) Plan() ([]UpgradePlan, error) { return ret, nil } -func (u *providerUpgrader) ApplyPlan(contract string) error { +func (u *providerUpgrader) ApplyPlan(opts UpgradeOptions, contract string) error { if contract != clusterv1.GroupVersion.Version { return errors.Errorf("current version of clusterctl could only upgrade to %s contract, requested %s", clusterv1.GroupVersion.Version, contract) } @@ -169,10 +175,10 @@ func (u *providerUpgrader) ApplyPlan(contract string) error { } // Do the upgrade - return u.doUpgrade(upgradePlan) + return u.doUpgrade(upgradePlan, opts) } -func (u *providerUpgrader) ApplyCustomPlan(upgradeItems ...UpgradeItem) error { +func (u *providerUpgrader) ApplyCustomPlan(opts UpgradeOptions, upgradeItems ...UpgradeItem) error { log := logf.Log log.Info("Performing upgrade...") @@ -184,7 +190,7 @@ func (u *providerUpgrader) ApplyCustomPlan(upgradeItems ...UpgradeItem) error { } // Do the upgrade - return u.doUpgrade(upgradePlan) + return u.doUpgrade(upgradePlan, opts) } // getUpgradePlan returns the upgrade plan for a specific set of providers/contract @@ -346,7 +352,7 @@ func (u *providerUpgrader) getUpgradeComponents(provider UpgradeItem) (repositor return components, nil } -func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error { +func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan, opts UpgradeOptions) error { // Check for multiple instances of the same provider if current contract is v1alpha3. if upgradePlan.Contract == clusterv1.GroupVersion.Version { if err := u.providerInventory.CheckSingleProviderInstance(); err != nil { @@ -380,6 +386,8 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error { } } + installQueue := []repository.Components{} + // Delete old providers and deploy new ones if necessary, i.e. there is a NextVersion. for _, upgradeItem := range providers { // If there is not a specified next version, skip it (we are already up-to-date). @@ -393,6 +401,8 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error { return err } + installQueue = append(installQueue, components) + // Delete the provider, preserving CRD, namespace and the inventory. if err := u.providerComponents.Delete(DeleteOptions{ Provider: upgradeItem.Provider, @@ -416,7 +426,7 @@ func (u *providerUpgrader) doUpgrade(upgradePlan *UpgradePlan) error { } } - return nil + return waitForProvidersReady(InstallOptions(opts), installQueue, u.proxy) } func (u *providerUpgrader) scaleDownProvider(provider clusterctlv1.Provider) error { diff --git a/cmd/clusterctl/client/cluster/upgrader_test.go b/cmd/clusterctl/client/cluster/upgrader_test.go index 3e0c962a9605..fad159ec74f4 100644 --- a/cmd/clusterctl/client/cluster/upgrader_test.go +++ b/cmd/clusterctl/client/cluster/upgrader_test.go @@ -888,6 +888,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) { contract string wantErr bool errorMsg string + opts UpgradeOptions }{ { name: "fails to upgrade to current contract when there are multiple instances of the core provider", @@ -924,6 +925,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) { contract: test.CurrentCAPIContract, wantErr: true, errorMsg: "detected multiple instances of the same provider", + opts: UpgradeOptions{}, }, { name: "fails to upgrade to current contract when there are multiple instances of the infra provider", @@ -960,6 +962,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) { contract: test.CurrentCAPIContract, wantErr: true, errorMsg: "detected multiple instances of the same provider", + opts: UpgradeOptions{}, }, } @@ -976,7 +979,7 @@ func Test_providerUpgrader_ApplyPlan(t *testing.T) { }, providerInventory: newInventoryClient(tt.fields.proxy, nil), } - err := u.ApplyPlan(tt.contract) + err := u.ApplyPlan(tt.opts, tt.contract) if tt.wantErr { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).Should(ContainSubstring(tt.errorMsg)) @@ -1002,6 +1005,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) { providersToUpgrade []UpgradeItem wantErr bool errorMsg string + opts UpgradeOptions }{ { name: "fails to upgrade to v1alpha4 when there are multiple instances of the core provider", @@ -1047,6 +1051,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) { }, wantErr: true, errorMsg: "invalid management cluster: there should a core provider, found 2", + opts: UpgradeOptions{}, }, { name: "fails to upgrade to v1alpha4 when there are multiple instances of the infra provider", @@ -1096,6 +1101,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) { }, wantErr: true, errorMsg: "detected multiple instances of the same provider", + opts: UpgradeOptions{}, }, } @@ -1112,7 +1118,7 @@ func Test_providerUpgrader_ApplyCustomPlan(t *testing.T) { }, providerInventory: newInventoryClient(tt.fields.proxy, nil), } - err := u.ApplyCustomPlan(tt.providersToUpgrade...) + err := u.ApplyCustomPlan(tt.opts, tt.providersToUpgrade...) if tt.wantErr { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).Should(ContainSubstring(tt.errorMsg)) diff --git a/cmd/clusterctl/client/upgrade.go b/cmd/clusterctl/client/upgrade.go index 2812d3027b85..eb224da11d70 100644 --- a/cmd/clusterctl/client/upgrade.go +++ b/cmd/clusterctl/client/upgrade.go @@ -18,6 +18,7 @@ package client import ( "strings" + "time" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -107,6 +108,12 @@ type ApplyUpgradeOptions struct { // InfrastructureProviders instance and versions (e.g. capa-system/aws:v0.5.0) to upgrade to. This field can be used as alternative to Contract. InfrastructureProviders []string + + // WaitProviders instructs the upgrade apply command to wait till the providers are successfully upgraded. + WaitProviders bool + + // WaitProviderTimeout sets the timeout per provider upgrade. + WaitProviderTimeout time.Duration } func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error { @@ -150,6 +157,11 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error { len(options.ControlPlaneProviders) > 0 || len(options.InfrastructureProviders) > 0 + opts := cluster.UpgradeOptions{ + WaitProviders: options.WaitProviders, + WaitProviderTimeout: options.WaitProviderTimeout, + } + // If we are upgrading a specific set of providers only, process the providers and call ApplyCustomPlan. if isCustomUpgrade { // Converts upgrade references back into an UpgradeItem. @@ -175,11 +187,11 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error { } // Execute the upgrade using the custom upgrade items - return clusterClient.ProviderUpgrader().ApplyCustomPlan(upgradeItems...) + return clusterClient.ProviderUpgrader().ApplyCustomPlan(opts, upgradeItems...) } // Otherwise we are upgrading a whole management cluster according to a clusterctl generated upgrade plan. - return clusterClient.ProviderUpgrader().ApplyPlan(options.Contract) + return clusterClient.ProviderUpgrader().ApplyPlan(opts, options.Contract) } func addUpgradeItems(upgradeItems []cluster.UpgradeItem, providerType clusterctlv1.ProviderType, providers ...string) ([]cluster.UpgradeItem, error) { diff --git a/cmd/clusterctl/cmd/upgrade_apply.go b/cmd/clusterctl/cmd/upgrade_apply.go index a54c733d1c83..ba569e9b04fa 100644 --- a/cmd/clusterctl/cmd/upgrade_apply.go +++ b/cmd/clusterctl/cmd/upgrade_apply.go @@ -17,6 +17,8 @@ limitations under the License. package cmd import ( + "time" + "github.com/pkg/errors" "github.com/spf13/cobra" @@ -31,6 +33,8 @@ type upgradeApplyOptions struct { bootstrapProviders []string controlPlaneProviders []string infrastructureProviders []string + waitProviders bool + waitProviderTimeout int } var ua = &upgradeApplyOptions{} @@ -73,6 +77,10 @@ func init() { "Bootstrap providers instance and versions (e.g. capi-kubeadm-bootstrap-system/kubeadm:v0.3.0) to upgrade to. This flag can be used as alternative to --contract.") upgradeApplyCmd.Flags().StringSliceVarP(&ua.controlPlaneProviders, "control-plane", "c", nil, "ControlPlane providers instance and versions (e.g. capi-kubeadm-control-plane-system/kubeadm:v0.3.0) to upgrade to. This flag can be used as alternative to --contract.") + upgradeApplyCmd.Flags().BoolVar(&ua.waitProviders, "wait-providers", false, + "Wait for providers to be upgraded.") + upgradeApplyCmd.Flags().IntVar(&ua.waitProviderTimeout, "wait-provider-timeout", 5*60, + "Wait timeout per provider upgrade in seconds. This value is ignored if --wait-providers is false") } func runUpgradeApply() error { @@ -100,5 +108,7 @@ func runUpgradeApply() error { BootstrapProviders: ua.bootstrapProviders, ControlPlaneProviders: ua.controlPlaneProviders, InfrastructureProviders: ua.infrastructureProviders, + WaitProviders: ua.waitProviders, + WaitProviderTimeout: time.Duration(ua.waitProviderTimeout) * time.Second, }) }