From ed0460bcd7e7e8b6c93b1a3a358a511bcd2f7afc Mon Sep 17 00:00:00 2001 From: Stephen Cahill Date: Mon, 25 Nov 2024 18:51:06 -0500 Subject: [PATCH 1/2] seedling: Add retry to clusterctl `UpgradeWithBinary` --- test/framework/clusterctl/client.go | 5 +++-- .../clusterctl/clusterctl_helpers.go | 21 ++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go index b1dcb25171e4..98e8b6bf8bcc 100644 --- a/test/framework/clusterctl/client.go +++ b/test/framework/clusterctl/client.go @@ -203,7 +203,7 @@ func Upgrade(ctx context.Context, input UpgradeInput) { } // UpgradeWithBinary calls clusterctl upgrade apply with the list of providers defined in the local repository. -func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) { +func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) error { if len(input.ClusterctlVariables) > 0 { outputPath := filepath.Join(filepath.Dir(input.ClusterctlConfigPath), fmt.Sprintf("clusterctl-upgrade-config-%s.yaml", input.ClusterName)) Expect(CopyAndAmendClusterctlConfig(ctx, CopyAndAmendClusterctlConfigInput{ @@ -227,8 +227,9 @@ func UpgradeWithBinary(ctx context.Context, binary string, input UpgradeInput) { if errors.As(err, &exitErr) { stdErr = string(exitErr.Stderr) } + return fmt.Errorf("failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr) } - Expect(err).ToNot(HaveOccurred(), "failed to run clusterctl upgrade apply:\nstdout:\n%s\nstderr:\n%s", string(out), stdErr) + return nil } func calculateClusterCtlUpgradeArgs(input UpgradeInput) []string { diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index c1db4c666cef..a87c47e433a4 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -196,12 +196,27 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen client := input.ClusterProxy.GetClient() if input.ClusterctlBinaryPath != "" { - UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput) + clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath) + Expect(err).ToNot(HaveOccurred()) + upgradeRetries := 1 + // Older versions of clusterctl may need to retry the upgrade process to allow for + // cert-manager CAs to become available before continuing. For newer versions of clusterctl + // this is addressed with https://github.com/kubernetes-sigs/cluster-api/pull/10513 + if clusterctlVersion.LT(semver.MustParse("1.7.0")) { + upgradeRetries = 2 + } + for i := range upgradeRetries { + err := UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput) + if err != nil && i < upgradeRetries-1 { + log.Logf("Failed to UpgradeWithBinary, retrying: %v", err) + continue + } + Expect(err).ToNot(HaveOccurred()) + break + } // Old versions of clusterctl may deploy CRDs, Mutating- and/or ValidatingWebhookConfigurations // before creating the new Certificate objects. This check ensures the CA's are up to date before // continuing. - clusterctlVersion, err := getClusterCtlVersion(input.ClusterctlBinaryPath) - Expect(err).ToNot(HaveOccurred()) if clusterctlVersion.LT(semver.MustParse("1.7.2")) { Eventually(func() error { return verifyCAInjection(ctx, client) From 2e1170a0d04b207c80f0d6fdd1dac2325df64947 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Wed, 4 Dec 2024 19:17:55 +0100 Subject: [PATCH 2/2] fix for go 1.21 --- test/framework/clusterctl/clusterctl_helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index a87c47e433a4..5e8b413794fb 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -205,7 +205,7 @@ func UpgradeManagementClusterAndWait(ctx context.Context, input UpgradeManagemen if clusterctlVersion.LT(semver.MustParse("1.7.0")) { upgradeRetries = 2 } - for i := range upgradeRetries { + for i := 0; i < upgradeRetries; i++ { err := UpgradeWithBinary(ctx, input.ClusterctlBinaryPath, upgradeInput) if err != nil && i < upgradeRetries-1 { log.Logf("Failed to UpgradeWithBinary, retrying: %v", err)